Merge branch 'master' into epoll1_h2proxy_tests_fix
diff --git a/doc/environment_variables.md b/doc/environment_variables.md
index f90f1d5..f775de1 100644
--- a/doc/environment_variables.md
+++ b/doc/environment_variables.md
@@ -58,6 +58,7 @@
completion queue
- round_robin - traces the round_robin load balancing policy
- pick_first - traces the pick first load balancing policy
+ - plugin_credentials - traces plugin credentials
- resource_quota - trace resource quota objects internals
- glb - traces the grpclb load balancer
- queue_pluck
diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h
index fab7d43..1de289f 100644
--- a/include/grpc/grpc.h
+++ b/include/grpc/grpc.h
@@ -313,7 +313,7 @@
void *reserved);
/** Ref a call.
- THREAD SAFETY: grpc_call_unref is thread-compatible */
+ THREAD SAFETY: grpc_call_ref is thread-compatible */
GRPCAPI void grpc_call_ref(grpc_call *call);
/** Unref a call.
diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h
index 2005e25..95b1447 100644
--- a/include/grpc/grpc_security.h
+++ b/include/grpc/grpc_security.h
@@ -249,19 +249,40 @@
void *reserved;
} grpc_auth_metadata_context;
+/** Maximum number of metadata entries returnable by a credentials plugin via
+ a synchronous return. */
+#define GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX 4
+
/** grpc_metadata_credentials plugin is an API user provided structure used to
create grpc_credentials objects that can be set on a channel (composed) or
a call. See grpc_credentials_metadata_create_from_plugin below.
The grpc client stack will call the get_metadata method of the plugin for
every call in scope for the credentials created from it. */
typedef struct {
- /** The implementation of this method has to be non-blocking.
- - context is the information that can be used by the plugin to create auth
- metadata.
- - cb is the callback that needs to be called when the metadata is ready.
- - user_data needs to be passed as the first parameter of the callback. */
- void (*get_metadata)(void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb, void *user_data);
+ /** The implementation of this method has to be non-blocking, but can
+ be performed synchronously or asynchronously.
+
+ If processing occurs synchronously, returns non-zero and populates
+ creds_md, num_creds_md, status, and error_details. In this case,
+ the caller takes ownership of the entries in creds_md and of
+ error_details. Note that if the plugin needs to return more than
+ GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX entries in creds_md, it must
+ return asynchronously.
+
+ If processing occurs asynchronously, returns zero and invokes \a cb
+ when processing is completed. \a user_data will be passed as the
+ first parameter of the callback. NOTE: \a cb MUST be invoked in a
+ different thread, not from the thread in which \a get_metadata() is
+ invoked.
+
+ \a context is the information that can be used by the plugin to create
+ auth metadata. */
+ int (*get_metadata)(
+ void *state, grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details);
/** Destroys the plugin state. */
void (*destroy)(void *state);
diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h
index 7e5f852..0f4316c 100644
--- a/include/grpc/impl/codegen/port_platform.h
+++ b/include/grpc/impl/codegen/port_platform.h
@@ -291,10 +291,6 @@
#endif
#ifdef _MSC_VER
-#ifdef _PYTHON_MSVC
-// The Python 3.5 Windows runtime is missing InetNtop
-#define GPR_WIN_INET_NTOP
-#endif // _PYTHON_MSVC
#if _MSC_VER < 1700
typedef __int8 int8_t;
typedef __int16 int16_t;
diff --git a/setup.py b/setup.py
index 1288241..90c9316 100644
--- a/setup.py
+++ b/setup.py
@@ -110,8 +110,6 @@
EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
else:
EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
- elif 'win32' in sys.platform:
- EXTRA_ENV_COMPILE_ARGS += ' -D_PYTHON_MSVC'
elif "linux" in sys.platform:
EXTRA_ENV_COMPILE_ARGS += ' -std=c++11 -std=gnu99 -fvisibility=hidden -fno-wrapv -fno-exceptions'
elif "darwin" in sys.platform:
@@ -163,7 +161,7 @@
# TODO(zyc): Re-enble c-ares on x64 and x86 windows after fixing the
# ares_library_init compilation issue
DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1), ('CARES_STATICLIB', 1),
- ('GRPC_ARES', 0),)
+ ('GRPC_ARES', 0), ('NTDDI_VERSION', 0x06000000),)
if '64bit' in platform.architecture()[0]:
DEFINE_MACROS += (('MS_WIN64', 1),)
elif sys.version_info >= (3, 5):
diff --git a/src/core/lib/iomgr/socket_utils_windows.c b/src/core/lib/iomgr/socket_utils_windows.c
index 2732c15..6e85e4b 100644
--- a/src/core/lib/iomgr/socket_utils_windows.c
+++ b/src/core/lib/iomgr/socket_utils_windows.c
@@ -26,12 +26,8 @@
#include <grpc/support/log.h>
const char *grpc_inet_ntop(int af, const void *src, char *dst, size_t size) {
-#ifdef GPR_WIN_INET_NTOP
- return inet_ntop(af, src, dst, size);
-#else
/* Windows InetNtopA wants a mutable ip pointer */
return InetNtopA(af, (void *)src, dst, size);
-#endif /* GPR_WIN_INET_NTOP */
}
#endif /* GRPC_WINDOWS_SOCKETUTILS */
diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c
index e9a7236..2472cf2 100644
--- a/src/core/lib/iomgr/timer_generic.c
+++ b/src/core/lib/iomgr/timer_generic.c
@@ -79,6 +79,125 @@
* Access to this is protected by g_shared_mutables.mu */
static timer_shard *g_shard_queue[NUM_SHARDS];
+#ifndef NDEBUG
+
+/* == Hash table for duplicate timer detection == */
+
+#define NUM_HASH_BUCKETS 1009 /* Prime number close to 1000 */
+
+static gpr_mu g_hash_mu[NUM_HASH_BUCKETS]; /* One mutex per bucket */
+static grpc_timer *g_timer_ht[NUM_HASH_BUCKETS] = {NULL};
+
+static void init_timer_ht() {
+ for (int i = 0; i < NUM_HASH_BUCKETS; i++) {
+ gpr_mu_init(&g_hash_mu[i]);
+ }
+}
+
+static bool is_in_ht(grpc_timer *t) {
+ size_t i = GPR_HASH_POINTER(t, NUM_HASH_BUCKETS);
+
+ gpr_mu_lock(&g_hash_mu[i]);
+ grpc_timer *p = g_timer_ht[i];
+ while (p != NULL && p != t) {
+ p = p->hash_table_next;
+ }
+ gpr_mu_unlock(&g_hash_mu[i]);
+
+ return (p == t);
+}
+
+static void add_to_ht(grpc_timer *t) {
+ GPR_ASSERT(!t->hash_table_next);
+ size_t i = GPR_HASH_POINTER(t, NUM_HASH_BUCKETS);
+
+ gpr_mu_lock(&g_hash_mu[i]);
+ grpc_timer *p = g_timer_ht[i];
+ while (p != NULL && p != t) {
+ p = p->hash_table_next;
+ }
+
+ if (p == t) {
+ grpc_closure *c = t->closure;
+ gpr_log(GPR_ERROR,
+ "** Duplicate timer (%p) being added. Closure: (%p), created at: "
+ "(%s:%d), scheduled at: (%s:%d) **",
+ t, c, c->file_created, c->line_created, c->file_initiated,
+ c->line_initiated);
+ abort();
+ }
+
+ /* Timer not present in the bucket. Insert at head of the list */
+ t->hash_table_next = g_timer_ht[i];
+ g_timer_ht[i] = t;
+ gpr_mu_unlock(&g_hash_mu[i]);
+}
+
+static void remove_from_ht(grpc_timer *t) {
+ size_t i = GPR_HASH_POINTER(t, NUM_HASH_BUCKETS);
+ bool removed = false;
+
+ gpr_mu_lock(&g_hash_mu[i]);
+ if (g_timer_ht[i] == t) {
+ g_timer_ht[i] = g_timer_ht[i]->hash_table_next;
+ removed = true;
+ } else if (g_timer_ht[i] != NULL) {
+ grpc_timer *p = g_timer_ht[i];
+ while (p->hash_table_next != NULL && p->hash_table_next != t) {
+ p = p->hash_table_next;
+ }
+
+ if (p->hash_table_next == t) {
+ p->hash_table_next = t->hash_table_next;
+ removed = true;
+ }
+ }
+ gpr_mu_unlock(&g_hash_mu[i]);
+
+ if (!removed) {
+ grpc_closure *c = t->closure;
+ gpr_log(GPR_ERROR,
+ "** Removing timer (%p) that is not added to hash table. Closure "
+ "(%p), created at: (%s:%d), scheduled at: (%s:%d) **",
+ t, c, c->file_created, c->line_created, c->file_initiated,
+ c->line_initiated);
+ abort();
+ }
+
+ t->hash_table_next = NULL;
+}
+
+/* If a timer is added to a timer shard (either heap or a list), it cannot
+ * be pending. A timer is added to hash table only-if it is added to the
+ * timer shard.
+ * Therefore, if timer->pending is false, it cannot be in hash table */
+static void validate_non_pending_timer(grpc_timer *t) {
+ if (!t->pending && is_in_ht(t)) {
+ grpc_closure *c = t->closure;
+ gpr_log(GPR_ERROR,
+ "** gpr_timer_cancel() called on a non-pending timer (%p) which "
+ "is in the hash table. Closure: (%p), created at: (%s:%d), "
+ "scheduled at: (%s:%d) **",
+ t, c, c->file_created, c->line_created, c->file_initiated,
+ c->line_initiated);
+ abort();
+ }
+}
+
+#define INIT_TIMER_HASH_TABLE() init_timer_ht()
+#define ADD_TO_HASH_TABLE(t) add_to_ht((t))
+#define REMOVE_FROM_HASH_TABLE(t) remove_from_ht((t))
+#define VALIDATE_NON_PENDING_TIMER(t) validate_non_pending_timer((t))
+
+#else
+
+#define INIT_TIMER_HASH_TABLE()
+#define ADD_TO_HASH_TABLE(t)
+#define REMOVE_FROM_HASH_TABLE(t)
+#define VALIDATE_NON_PENDING_TIMER(t)
+
+#endif
+
/* Thread local variable that stores the deadline of the next timer the thread
* has last-seen. This is an optimization to prevent the thread from checking
* shared_mutables.min_timer (which requires acquiring shared_mutables.mu lock,
@@ -175,6 +294,8 @@
shard->min_deadline = compute_min_deadline(shard);
g_shard_queue[i] = shard;
}
+
+ INIT_TIMER_HASH_TABLE();
}
void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) {
@@ -245,6 +366,10 @@
timer->closure = closure;
gpr_atm deadline_atm = timer->deadline = timespec_to_atm_round_up(deadline);
+#ifndef NDEBUG
+ timer->hash_table_next = NULL;
+#endif
+
if (GRPC_TRACER_ON(grpc_timer_trace)) {
gpr_log(GPR_DEBUG, "TIMER %p: SET %" PRId64 ".%09d [%" PRIdPTR
"] now %" PRId64 ".%09d [%" PRIdPTR "] call %p[%p]",
@@ -272,6 +397,9 @@
grpc_time_averaged_stats_add_sample(&shard->stats,
ts_to_dbl(gpr_time_sub(deadline, now)));
+
+ ADD_TO_HASH_TABLE(timer);
+
if (deadline_atm < shard->queue_deadline_cap) {
is_first_timer = grpc_timer_heap_add(&shard->heap, timer);
} else {
@@ -333,7 +461,10 @@
gpr_log(GPR_DEBUG, "TIMER %p: CANCEL pending=%s", timer,
timer->pending ? "true" : "false");
}
+
if (timer->pending) {
+ REMOVE_FROM_HASH_TABLE(timer);
+
GRPC_CLOSURE_SCHED(exec_ctx, timer->closure, GRPC_ERROR_CANCELLED);
timer->pending = false;
if (timer->heap_index == INVALID_HEAP_INDEX) {
@@ -341,6 +472,8 @@
} else {
grpc_timer_heap_remove(&shard->heap, timer);
}
+ } else {
+ VALIDATE_NON_PENDING_TIMER(timer);
}
gpr_mu_unlock(&shard->mu);
}
@@ -424,6 +557,7 @@
grpc_timer *timer;
gpr_mu_lock(&shard->mu);
while ((timer = pop_one(shard, now))) {
+ REMOVE_FROM_HASH_TABLE(timer);
GRPC_CLOSURE_SCHED(exec_ctx, timer->closure, GRPC_ERROR_REF(error));
n++;
}
diff --git a/src/core/lib/iomgr/timer_generic.h b/src/core/lib/iomgr/timer_generic.h
index 72a4ac1..f0597f6 100644
--- a/src/core/lib/iomgr/timer_generic.h
+++ b/src/core/lib/iomgr/timer_generic.h
@@ -29,6 +29,9 @@
struct grpc_timer *next;
struct grpc_timer *prev;
grpc_closure *closure;
+#ifndef NDEBUG
+ struct grpc_timer *hash_table_next;
+#endif
};
#endif /* GRPC_CORE_LIB_IOMGR_TIMER_GENERIC_H */
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
index 09fd60a..b67ff48 100644
--- a/src/core/lib/security/credentials/composite/composite_credentials.c
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -87,6 +87,7 @@
ctx->on_request_metadata = on_request_metadata;
GRPC_CLOSURE_INIT(&ctx->internal_on_request_metadata,
composite_call_metadata_cb, ctx, grpc_schedule_on_exec_ctx);
+ bool synchronous = true;
while (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
grpc_call_credentials *inner_creds =
ctx->composite_creds->inner.creds_array[ctx->creds_index++];
@@ -95,19 +96,12 @@
ctx->md_array, &ctx->internal_on_request_metadata, error)) {
if (*error != GRPC_ERROR_NONE) break;
} else {
+ synchronous = false; // Async return.
break;
}
}
- // If we got through all creds synchronously or we got a synchronous
- // error on one of them, return synchronously.
- if (ctx->creds_index == ctx->composite_creds->inner.num_creds ||
- *error != GRPC_ERROR_NONE) {
- gpr_free(ctx);
- return true;
- }
- // At least one inner cred is returning asynchronously, so we'll
- // return asynchronously as well.
- return false;
+ if (synchronous) gpr_free(ctx);
+ return synchronous;
}
static void composite_call_cancel_get_request_metadata(
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
index 73e0c23..ee20241 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.c
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -31,6 +31,9 @@
#include "src/core/lib/surface/api_trace.h"
#include "src/core/lib/surface/validate_metadata.h"
+grpc_tracer_flag grpc_plugin_credentials_trace =
+ GRPC_TRACER_INITIALIZER(false, "plugin_credentials");
+
static void plugin_destruct(grpc_exec_ctx *exec_ctx,
grpc_call_credentials *creds) {
grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
@@ -53,6 +56,62 @@
}
}
+// Checks if the request has been cancelled.
+// If not, removes it from the pending list, so that it cannot be
+// cancelled out from under us.
+// When this returns, r->cancelled indicates whether the request was
+// cancelled before completion.
+static void pending_request_complete(
+ grpc_exec_ctx *exec_ctx, grpc_plugin_credentials_pending_request *r) {
+ gpr_mu_lock(&r->creds->mu);
+ if (!r->cancelled) pending_request_remove_locked(r->creds, r);
+ gpr_mu_unlock(&r->creds->mu);
+ // Ref to credentials not needed anymore.
+ grpc_call_credentials_unref(exec_ctx, &r->creds->base);
+}
+
+static grpc_error *process_plugin_result(
+ grpc_exec_ctx *exec_ctx, grpc_plugin_credentials_pending_request *r,
+ const grpc_metadata *md, size_t num_md, grpc_status_code status,
+ const char *error_details) {
+ grpc_error *error = GRPC_ERROR_NONE;
+ if (status != GRPC_STATUS_OK) {
+ char *msg;
+ gpr_asprintf(&msg, "Getting metadata from plugin failed with error: %s",
+ error_details);
+ error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg);
+ gpr_free(msg);
+ } else {
+ bool seen_illegal_header = false;
+ for (size_t i = 0; i < num_md; ++i) {
+ if (!GRPC_LOG_IF_ERROR("validate_metadata_from_plugin",
+ grpc_validate_header_key_is_legal(md[i].key))) {
+ seen_illegal_header = true;
+ break;
+ } else if (!grpc_is_binary_header(md[i].key) &&
+ !GRPC_LOG_IF_ERROR(
+ "validate_metadata_from_plugin",
+ grpc_validate_header_nonbin_value_is_legal(md[i].value))) {
+ gpr_log(GPR_ERROR, "Plugin added invalid metadata value.");
+ seen_illegal_header = true;
+ break;
+ }
+ }
+ if (seen_illegal_header) {
+ error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal metadata");
+ } else {
+ for (size_t i = 0; i < num_md; ++i) {
+ grpc_mdelem mdelem = grpc_mdelem_from_slices(
+ exec_ctx, grpc_slice_ref_internal(md[i].key),
+ grpc_slice_ref_internal(md[i].value));
+ grpc_credentials_mdelem_array_add(r->md_array, mdelem);
+ GRPC_MDELEM_UNREF(exec_ctx, mdelem);
+ }
+ }
+ }
+ return error;
+}
+
static void plugin_md_request_metadata_ready(void *request,
const grpc_metadata *md,
size_t num_md,
@@ -64,54 +123,24 @@
NULL, NULL);
grpc_plugin_credentials_pending_request *r =
(grpc_plugin_credentials_pending_request *)request;
- // Check if the request has been cancelled.
- // If not, remove it from the pending list, so that it cannot be
- // cancelled out from under us.
- gpr_mu_lock(&r->creds->mu);
- if (!r->cancelled) pending_request_remove_locked(r->creds, r);
- gpr_mu_unlock(&r->creds->mu);
- grpc_call_credentials_unref(&exec_ctx, &r->creds->base);
+ if (GRPC_TRACER_ON(grpc_plugin_credentials_trace)) {
+ gpr_log(GPR_INFO,
+ "plugin_credentials[%p]: request %p: plugin returned "
+ "asynchronously",
+ r->creds, r);
+ }
+ // Remove request from pending list if not previously cancelled.
+ pending_request_complete(&exec_ctx, r);
// If it has not been cancelled, process it.
if (!r->cancelled) {
- if (status != GRPC_STATUS_OK) {
- char *msg;
- gpr_asprintf(&msg, "Getting metadata from plugin failed with error: %s",
- error_details);
- GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata,
- GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg));
- gpr_free(msg);
- } else {
- bool seen_illegal_header = false;
- for (size_t i = 0; i < num_md; ++i) {
- if (!GRPC_LOG_IF_ERROR("validate_metadata_from_plugin",
- grpc_validate_header_key_is_legal(md[i].key))) {
- seen_illegal_header = true;
- break;
- } else if (!grpc_is_binary_header(md[i].key) &&
- !GRPC_LOG_IF_ERROR(
- "validate_metadata_from_plugin",
- grpc_validate_header_nonbin_value_is_legal(
- md[i].value))) {
- gpr_log(GPR_ERROR, "Plugin added invalid metadata value.");
- seen_illegal_header = true;
- break;
- }
- }
- if (seen_illegal_header) {
- GRPC_CLOSURE_SCHED(
- &exec_ctx, r->on_request_metadata,
- GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal metadata"));
- } else {
- for (size_t i = 0; i < num_md; ++i) {
- grpc_mdelem mdelem = grpc_mdelem_from_slices(
- &exec_ctx, grpc_slice_ref_internal(md[i].key),
- grpc_slice_ref_internal(md[i].value));
- grpc_credentials_mdelem_array_add(r->md_array, mdelem);
- GRPC_MDELEM_UNREF(&exec_ctx, mdelem);
- }
- GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata, GRPC_ERROR_NONE);
- }
- }
+ grpc_error *error =
+ process_plugin_result(&exec_ctx, r, md, num_md, status, error_details);
+ GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata, error);
+ } else if (GRPC_TRACER_ON(grpc_plugin_credentials_trace)) {
+ gpr_log(GPR_INFO,
+ "plugin_credentials[%p]: request %p: plugin was previously "
+ "cancelled",
+ r->creds, r);
}
gpr_free(r);
grpc_exec_ctx_finish(&exec_ctx);
@@ -125,6 +154,7 @@
grpc_closure *on_request_metadata,
grpc_error **error) {
grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+ bool retval = true; // Synchronous return.
if (c->plugin.get_metadata != NULL) {
// Create pending_request object.
grpc_plugin_credentials_pending_request *pending_request =
@@ -142,12 +172,60 @@
c->pending_requests = pending_request;
gpr_mu_unlock(&c->mu);
// Invoke the plugin. The callback holds a ref to us.
+ if (GRPC_TRACER_ON(grpc_plugin_credentials_trace)) {
+ gpr_log(GPR_INFO, "plugin_credentials[%p]: request %p: invoking plugin",
+ c, pending_request);
+ }
grpc_call_credentials_ref(creds);
- c->plugin.get_metadata(c->plugin.state, context,
- plugin_md_request_metadata_ready, pending_request);
- return false;
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX];
+ size_t num_creds_md = 0;
+ grpc_status_code status = GRPC_STATUS_OK;
+ const char *error_details = NULL;
+ if (!c->plugin.get_metadata(c->plugin.state, context,
+ plugin_md_request_metadata_ready,
+ pending_request, creds_md, &num_creds_md,
+ &status, &error_details)) {
+ if (GRPC_TRACER_ON(grpc_plugin_credentials_trace)) {
+ gpr_log(GPR_INFO,
+ "plugin_credentials[%p]: request %p: plugin will return "
+ "asynchronously",
+ c, pending_request);
+ }
+ return false; // Asynchronous return.
+ }
+ // Returned synchronously.
+ // Remove request from pending list if not previously cancelled.
+ pending_request_complete(exec_ctx, pending_request);
+ // If the request was cancelled, the error will have been returned
+ // asynchronously by plugin_cancel_get_request_metadata(), so return
+ // false. Otherwise, process the result.
+ if (pending_request->cancelled) {
+ if (GRPC_TRACER_ON(grpc_plugin_credentials_trace)) {
+ gpr_log(GPR_INFO,
+ "plugin_credentials[%p]: request %p was cancelled, error "
+ "will be returned asynchronously",
+ c, pending_request);
+ }
+ retval = false;
+ } else {
+ if (GRPC_TRACER_ON(grpc_plugin_credentials_trace)) {
+ gpr_log(GPR_INFO,
+ "plugin_credentials[%p]: request %p: plugin returned "
+ "synchronously",
+ c, pending_request);
+ }
+ *error = process_plugin_result(exec_ctx, pending_request, creds_md,
+ num_creds_md, status, error_details);
+ }
+ // Clean up.
+ for (size_t i = 0; i < num_creds_md; ++i) {
+ grpc_slice_unref_internal(exec_ctx, creds_md[i].key);
+ grpc_slice_unref_internal(exec_ctx, creds_md[i].value);
+ }
+ gpr_free((void *)error_details);
+ gpr_free(pending_request);
}
- return true;
+ return retval;
}
static void plugin_cancel_get_request_metadata(
@@ -159,6 +237,10 @@
c->pending_requests;
pending_request != NULL; pending_request = pending_request->next) {
if (pending_request->md_array == md_array) {
+ if (GRPC_TRACER_ON(grpc_plugin_credentials_trace)) {
+ gpr_log(GPR_INFO, "plugin_credentials[%p]: cancelling request %p", c,
+ pending_request);
+ }
pending_request->cancelled = true;
GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata,
GRPC_ERROR_REF(error));
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
index 57266d5..f56df9e 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.h
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -21,6 +21,8 @@
#include "src/core/lib/security/credentials/credentials.h"
+extern grpc_tracer_flag grpc_plugin_credentials_trace;
+
struct grpc_plugin_credentials;
typedef struct grpc_plugin_credentials_pending_request {
diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c
index 2366c24..8fbde3d 100644
--- a/src/core/lib/surface/init_secure.c
+++ b/src/core/lib/surface/init_secure.c
@@ -25,6 +25,7 @@
#include "src/core/lib/debug/trace.h"
#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/credentials/plugin/plugin_credentials.h"
#include "src/core/lib/security/transport/auth_filters.h"
#include "src/core/lib/security/transport/secure_endpoint.h"
#include "src/core/lib/security/transport/security_connector.h"
@@ -84,4 +85,7 @@
maybe_prepend_server_auth_filter, NULL);
}
-void grpc_security_init() { grpc_security_register_handshaker_factories(); }
+void grpc_security_init() {
+ grpc_security_register_handshaker_factories();
+ grpc_register_tracer(&grpc_plugin_credentials_trace);
+}
diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc
index 057a058..13bbc30 100644
--- a/src/cpp/client/secure_credentials.cc
+++ b/src/cpp/client/secure_credentials.cc
@@ -21,6 +21,7 @@
#include <grpc++/impl/grpc_library.h>
#include <grpc++/support/channel_arguments.h>
#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
#include "src/cpp/client/create_channel_internal.h"
#include "src/cpp/common/secure_auth_context.h"
@@ -150,6 +151,18 @@
return nullptr;
}
+std::shared_ptr<CallCredentials> CompositeCallCredentials(
+ const std::shared_ptr<CallCredentials>& creds1,
+ const std::shared_ptr<CallCredentials>& creds2) {
+ SecureCallCredentials* s_creds1 = creds1->AsSecureCredentials();
+ SecureCallCredentials* s_creds2 = creds2->AsSecureCredentials();
+ if (s_creds1 != nullptr && s_creds2 != nullptr) {
+ return WrapCallCredentials(grpc_composite_call_credentials_create(
+ s_creds1->GetRawCreds(), s_creds2->GetRawCreds(), nullptr));
+ }
+ return nullptr;
+}
+
void MetadataCredentialsPluginWrapper::Destroy(void* wrapper) {
if (wrapper == nullptr) return;
MetadataCredentialsPluginWrapper* w =
@@ -157,28 +170,50 @@
delete w;
}
-void MetadataCredentialsPluginWrapper::GetMetadata(
+int MetadataCredentialsPluginWrapper::GetMetadata(
void* wrapper, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb, void* user_data) {
+ grpc_credentials_plugin_metadata_cb cb, void* user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t* num_creds_md, grpc_status_code* status,
+ const char** error_details) {
GPR_ASSERT(wrapper);
MetadataCredentialsPluginWrapper* w =
reinterpret_cast<MetadataCredentialsPluginWrapper*>(wrapper);
if (!w->plugin_) {
- cb(user_data, NULL, 0, GRPC_STATUS_OK, NULL);
- return;
+ *num_creds_md = 0;
+ *status = GRPC_STATUS_OK;
+ *error_details = nullptr;
+ return true;
}
if (w->plugin_->IsBlocking()) {
+ // Asynchronous return.
w->thread_pool_->Add(
std::bind(&MetadataCredentialsPluginWrapper::InvokePlugin, w, context,
- cb, user_data));
+ cb, user_data, nullptr, nullptr, nullptr, nullptr));
+ return 0;
} else {
- w->InvokePlugin(context, cb, user_data);
+ // Synchronous return.
+ w->InvokePlugin(context, cb, user_data, creds_md, num_creds_md, status,
+ error_details);
+ return 1;
}
}
+namespace {
+
+void UnrefMetadata(const std::vector<grpc_metadata>& md) {
+ for (auto it = md.begin(); it != md.end(); ++it) {
+ grpc_slice_unref(it->key);
+ grpc_slice_unref(it->value);
+ }
+}
+
+} // namespace
+
void MetadataCredentialsPluginWrapper::InvokePlugin(
grpc_auth_metadata_context context, grpc_credentials_plugin_metadata_cb cb,
- void* user_data) {
+ void* user_data, grpc_metadata creds_md[4], size_t* num_creds_md,
+ grpc_status_code* status_code, const char** error_details) {
std::multimap<grpc::string, grpc::string> metadata;
// const_cast is safe since the SecureAuthContext does not take owndership and
@@ -196,12 +231,31 @@
md_entry.flags = 0;
md.push_back(md_entry);
}
- cb(user_data, md.empty() ? nullptr : &md[0], md.size(),
- static_cast<grpc_status_code>(status.error_code()),
- status.error_message().c_str());
- for (auto it = md.begin(); it != md.end(); ++it) {
- grpc_slice_unref(it->key);
- grpc_slice_unref(it->value);
+ if (creds_md != nullptr) {
+ // Synchronous return.
+ if (md.size() > GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX) {
+ *num_creds_md = 0;
+ *status_code = GRPC_STATUS_INTERNAL;
+ *error_details = gpr_strdup(
+ "blocking plugin credentials returned too many metadata keys");
+ UnrefMetadata(md);
+ } else {
+ for (const auto& elem : md) {
+ creds_md[*num_creds_md].key = elem.key;
+ creds_md[*num_creds_md].value = elem.value;
+ creds_md[*num_creds_md].flags = elem.flags;
+ ++(*num_creds_md);
+ }
+ *status_code = static_cast<grpc_status_code>(status.error_code());
+ *error_details =
+ status.ok() ? nullptr : gpr_strdup(status.error_message().c_str());
+ }
+ } else {
+ // Asynchronous return.
+ cb(user_data, md.empty() ? nullptr : &md[0], md.size(),
+ static_cast<grpc_status_code>(status.error_code()),
+ status.error_message().c_str());
+ UnrefMetadata(md);
}
}
diff --git a/src/cpp/client/secure_credentials.h b/src/cpp/client/secure_credentials.h
index 66547c7..ed9afb3 100644
--- a/src/cpp/client/secure_credentials.h
+++ b/src/cpp/client/secure_credentials.h
@@ -58,16 +58,23 @@
class MetadataCredentialsPluginWrapper final : private GrpcLibraryCodegen {
public:
static void Destroy(void* wrapper);
- static void GetMetadata(void* wrapper, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb,
- void* user_data);
+ static int GetMetadata(
+ void* wrapper, grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void* user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t* num_creds_md, grpc_status_code* status,
+ const char** error_details);
explicit MetadataCredentialsPluginWrapper(
std::unique_ptr<MetadataCredentialsPlugin> plugin);
private:
- void InvokePlugin(grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb, void* user_data);
+ void InvokePlugin(
+ grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void* user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t* num_creds_md, grpc_status_code* status_code,
+ const char** error_details);
std::unique_ptr<ThreadPoolInterface> thread_pool_;
std::unique_ptr<MetadataCredentialsPlugin> plugin_;
};
diff --git a/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs b/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs
index b56bdbb..a8cb357 100644
--- a/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs
+++ b/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs
@@ -61,12 +61,9 @@
try
{
- var context = new AuthInterceptorContext(Marshal.PtrToStringAnsi(serviceUrlPtr),
- Marshal.PtrToStringAnsi(methodNamePtr));
- // Don't await, we are in a native callback and need to return.
- #pragma warning disable 4014
- GetMetadataAsync(context, callbackPtr, userDataPtr);
- #pragma warning restore 4014
+ var context = new AuthInterceptorContext(Marshal.PtrToStringAnsi(serviceUrlPtr), Marshal.PtrToStringAnsi(methodNamePtr));
+ // Make a guarantee that credentials_notify_from_plugin is invoked async to be compliant with c-core API.
+ ThreadPool.QueueUserWorkItem(async (stateInfo) => await GetMetadataAsync(context, callbackPtr, userDataPtr));
}
catch (Exception e)
{
diff --git a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
index e81157c..eba6276 100644
--- a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
@@ -90,6 +90,54 @@
}
[Test]
+ public async Task MetadataCredentials_Composed()
+ {
+ var first = CallCredentials.FromInterceptor(new AsyncAuthInterceptor((context, metadata) => {
+ // Attempt to exercise the case where async callback is inlineable/synchronously-runnable.
+ metadata.Add("first_authorization", "FIRST_SECRET_TOKEN");
+ return TaskUtils.CompletedTask;
+ }));
+ var second = CallCredentials.FromInterceptor(new AsyncAuthInterceptor((context, metadata) => {
+ metadata.Add("second_authorization", "SECOND_SECRET_TOKEN");
+ return TaskUtils.CompletedTask;
+ }));
+ var third = CallCredentials.FromInterceptor(new AsyncAuthInterceptor((context, metadata) => {
+ metadata.Add("third_authorization", "THIRD_SECRET_TOKEN");
+ return TaskUtils.CompletedTask;
+ }));
+ var channelCredentials = ChannelCredentials.Create(TestCredentials.CreateSslCredentials(),
+ CallCredentials.Compose(first, second, third));
+ channel = new Channel(Host, server.Ports.Single().BoundPort, channelCredentials, options);
+ var client = new TestService.TestServiceClient(channel);
+ var call = client.StreamingOutputCall(new StreamingOutputCallRequest { });
+ Assert.IsTrue(await call.ResponseStream.MoveNext());
+ Assert.IsFalse(await call.ResponseStream.MoveNext());
+ }
+
+ [Test]
+ public async Task MetadataCredentials_ComposedPerCall()
+ {
+ channel = new Channel(Host, server.Ports.Single().BoundPort, TestCredentials.CreateSslCredentials(), options);
+ var client = new TestService.TestServiceClient(channel);
+ var first = CallCredentials.FromInterceptor(new AsyncAuthInterceptor((context, metadata) => {
+ metadata.Add("first_authorization", "FIRST_SECRET_TOKEN");
+ return TaskUtils.CompletedTask;
+ }));
+ var second = CallCredentials.FromInterceptor(new AsyncAuthInterceptor((context, metadata) => {
+ metadata.Add("second_authorization", "SECOND_SECRET_TOKEN");
+ return TaskUtils.CompletedTask;
+ }));
+ var third = CallCredentials.FromInterceptor(new AsyncAuthInterceptor((context, metadata) => {
+ metadata.Add("third_authorization", "THIRD_SECRET_TOKEN");
+ return TaskUtils.CompletedTask;
+ }));
+ var call = client.StreamingOutputCall(new StreamingOutputCallRequest{ },
+ new CallOptions(credentials: CallCredentials.Compose(first, second, third)));
+ Assert.IsTrue(await call.ResponseStream.MoveNext());
+ Assert.IsFalse(await call.ResponseStream.MoveNext());
+ }
+
+ [Test]
public void MetadataCredentials_InterceptorLeavesMetadataEmpty()
{
var channelCredentials = ChannelCredentials.Create(TestCredentials.CreateSslCredentials(),
@@ -125,6 +173,17 @@
Assert.AreEqual("SECRET_TOKEN", authToken);
return Task.FromResult(new SimpleResponse());
}
+
+ public override async Task StreamingOutputCall(StreamingOutputCallRequest request, IServerStreamWriter<StreamingOutputCallResponse> responseStream, ServerCallContext context)
+ {
+ var first = context.RequestHeaders.First((entry) => entry.Key == "first_authorization").Value;
+ Assert.AreEqual("FIRST_SECRET_TOKEN", first);
+ var second = context.RequestHeaders.First((entry) => entry.Key == "second_authorization").Value;
+ Assert.AreEqual("SECOND_SECRET_TOKEN", second);
+ var third = context.RequestHeaders.First((entry) => entry.Key == "third_authorization").Value;
+ Assert.AreEqual("THIRD_SECRET_TOKEN", third);
+ await responseStream.WriteAsync(new StreamingOutputCallResponse());
+ }
}
}
}
diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c
index aebce36..92291f9 100644
--- a/src/csharp/ext/grpc_csharp_ext.c
+++ b/src/csharp/ext/grpc_csharp_ext.c
@@ -1023,13 +1023,17 @@
grpc_credentials_plugin_metadata_cb cb, void *user_data,
int32_t is_destroy);
-static void grpcsharp_get_metadata_handler(
+static int grpcsharp_get_metadata_handler(
void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb, void *user_data) {
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details) {
grpcsharp_metadata_interceptor_func interceptor =
(grpcsharp_metadata_interceptor_func)(intptr_t)state;
interceptor(state, context.service_url, context.method_name, cb, user_data,
0);
+ return 0; /* Asynchronous return. */
}
static void grpcsharp_metadata_credentials_destroy_handler(void *state) {
diff --git a/src/node/ext/call_credentials.cc b/src/node/ext/call_credentials.cc
index 4cf3e56..0644a81 100644
--- a/src/node/ext/call_credentials.cc
+++ b/src/node/ext/call_credentials.cc
@@ -238,9 +238,12 @@
}
}
-void plugin_get_metadata(void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb,
- void *user_data) {
+int plugin_get_metadata(
+ void *state, grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details) {
plugin_state *p_state = reinterpret_cast<plugin_state *>(state);
plugin_callback_data *data = new plugin_callback_data;
data->service_url = context.service_url;
@@ -252,6 +255,7 @@
uv_mutex_unlock(&p_state->plugin_mutex);
uv_async_send(&p_state->plugin_async);
+ return 0; // Async processing.
}
void plugin_uv_close_cb(uv_handle_t *handle) {
diff --git a/src/node/ext/call_credentials.h b/src/node/ext/call_credentials.h
index adcff84..3a54bbf 100644
--- a/src/node/ext/call_credentials.h
+++ b/src/node/ext/call_credentials.h
@@ -75,9 +75,11 @@
uv_async_t plugin_async;
} plugin_state;
-void plugin_get_metadata(void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb,
- void *user_data);
+int plugin_get_metadata(
+ void *state, grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status, const char **error_details);
void plugin_destroy_state(void *state);
diff --git a/src/php/ext/grpc/call_credentials.c b/src/php/ext/grpc/call_credentials.c
index 1eee864..a395d53 100644
--- a/src/php/ext/grpc/call_credentials.c
+++ b/src/php/ext/grpc/call_credentials.c
@@ -35,6 +35,7 @@
#include <grpc/grpc.h>
#include <grpc/grpc_security.h>
+#include <grpc/support/string_util.h>
zend_class_entry *grpc_ce_call_credentials;
#if PHP_MAJOR_VERSION >= 7
@@ -143,9 +144,12 @@
}
/* Callback function for plugin creds API */
-void plugin_get_metadata(void *ptr, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb,
- void *user_data) {
+int plugin_get_metadata(
+ void *ptr, grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details) {
TSRMLS_FETCH();
plugin_state *state = (plugin_state *)ptr;
@@ -175,15 +179,19 @@
/* call the user callback function */
zend_call_function(state->fci, state->fci_cache TSRMLS_CC);
- grpc_status_code code = GRPC_STATUS_OK;
+ *num_creds_md = 0;
+ *status = GRPC_STATUS_OK;
+ *error_details = NULL;
+
grpc_metadata_array metadata;
- bool cleanup = true;
if (retval == NULL || Z_TYPE_P(retval) != IS_ARRAY) {
- cleanup = false;
- code = GRPC_STATUS_INVALID_ARGUMENT;
- } else if (!create_metadata_array(retval, &metadata)) {
- code = GRPC_STATUS_INVALID_ARGUMENT;
+ *status = GRPC_STATUS_INVALID_ARGUMENT;
+ return true; // Synchronous return.
+ }
+ if (!create_metadata_array(retval, &metadata)) {
+ *status = GRPC_STATUS_INVALID_ARGUMENT;
+ return true; // Synchronous return.
}
if (retval != NULL) {
@@ -197,14 +205,24 @@
#endif
}
- /* Pass control back to core */
- cb(user_data, metadata.metadata, metadata.count, code, NULL);
- if (cleanup) {
- for (int i = 0; i < metadata.count; i++) {
+ if (metadata.count > GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX) {
+ *status = GRPC_STATUS_INTERNAL;
+ *error_details = gpr_strdup(
+ "PHP plugin credentials returned too many metadata entries");
+ for (size_t i = 0; i < metadata.count; i++) {
+ // TODO(stanleycheung): Why don't we need to unref the key here?
grpc_slice_unref(metadata.metadata[i].value);
}
- grpc_metadata_array_destroy(&metadata);
+ } else {
+ // Return data to core.
+ *num_creds_md = metadata.count;
+ for (size_t i = 0; i < metadata.count; ++i) {
+ creds_md[i] = metadata.metadata[i];
+ }
}
+
+ grpc_metadata_array_destroy(&metadata);
+ return true; // Synchronous return.
}
/* Cleanup function for plugin creds API */
diff --git a/src/php/ext/grpc/call_credentials.h b/src/php/ext/grpc/call_credentials.h
index 9be8763..663cc68 100755
--- a/src/php/ext/grpc/call_credentials.h
+++ b/src/php/ext/grpc/call_credentials.h
@@ -65,9 +65,12 @@
} plugin_state;
/* Callback function for plugin creds API */
-void plugin_get_metadata(void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb,
- void *user_data);
+int plugin_get_metadata(
+ void *ptr, grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details);
/* Cleanup function for plugin creds API */
void plugin_destroy_state(void *ptr);
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi
index a0e69dd..41975cb 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi
@@ -41,7 +41,8 @@
cdef object plugin_callback
cdef bytes plugin_name
- cdef grpc_metadata_credentials_plugin make_c_plugin(self)
+
+cdef grpc_metadata_credentials_plugin _c_plugin(CredentialsMetadataPlugin plugin)
cdef class AuthMetadataContext:
@@ -49,8 +50,11 @@
cdef grpc_auth_metadata_context context
-cdef void plugin_get_metadata(
+cdef int plugin_get_metadata(
void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb, void *user_data) with gil
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details) with gil
cdef void plugin_destroy_c_plugin_state(void *state) with gil
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
index 57816f1..0fabda1 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi
@@ -14,6 +14,7 @@
cimport cpython
+import threading
import traceback
@@ -89,20 +90,20 @@
self.plugin_callback = plugin_callback
self.plugin_name = name
- @staticmethod
- cdef grpc_metadata_credentials_plugin make_c_plugin(self):
- cdef grpc_metadata_credentials_plugin result
- result.get_metadata = plugin_get_metadata
- result.destroy = plugin_destroy_c_plugin_state
- result.state = <void *>self
- result.type = self.plugin_name
- cpython.Py_INCREF(self)
- return result
-
def __dealloc__(self):
grpc_shutdown()
+cdef grpc_metadata_credentials_plugin _c_plugin(CredentialsMetadataPlugin plugin):
+ cdef grpc_metadata_credentials_plugin c_plugin
+ c_plugin.get_metadata = plugin_get_metadata
+ c_plugin.destroy = plugin_destroy_c_plugin_state
+ c_plugin.state = <void *>plugin
+ c_plugin.type = plugin.plugin_name
+ cpython.Py_INCREF(plugin)
+ return c_plugin
+
+
cdef class AuthMetadataContext:
def __cinit__(self):
@@ -122,9 +123,12 @@
grpc_shutdown()
-cdef void plugin_get_metadata(
+cdef int plugin_get_metadata(
void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb, void *user_data) with gil:
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details) with gil:
called_flag = [False]
def python_callback(
Metadata metadata, grpc_status_code status,
@@ -134,12 +138,15 @@
cdef CredentialsMetadataPlugin self = <CredentialsMetadataPlugin>state
cdef AuthMetadataContext cy_context = AuthMetadataContext()
cy_context.context = context
- try:
- self.plugin_callback(cy_context, python_callback)
- except Exception as error:
- if not called_flag[0]:
- cb(user_data, NULL, 0, StatusCode.unknown,
- traceback.format_exc().encode())
+ def async_callback():
+ try:
+ self.plugin_callback(cy_context, python_callback)
+ except Exception as error:
+ if not called_flag[0]:
+ cb(user_data, NULL, 0, StatusCode.unknown,
+ traceback.format_exc().encode())
+ threading.Thread(group=None, target=async_callback).start()
+ return 0 # Asynchronous return
cdef void plugin_destroy_c_plugin_state(void *state) with gil:
cpython.Py_DECREF(<CredentialsMetadataPlugin>state)
@@ -239,7 +246,7 @@
def call_credentials_metadata_plugin(CredentialsMetadataPlugin plugin):
cdef CallCredentials credentials = CallCredentials()
- cdef grpc_metadata_credentials_plugin c_plugin = plugin.make_c_plugin()
+ cdef grpc_metadata_credentials_plugin c_plugin = _c_plugin(plugin)
with nogil:
credentials.c_credentials = (
grpc_metadata_credentials_create_from_plugin(c_plugin, NULL))
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
index 840af5c..f115106 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
@@ -375,6 +375,10 @@
cdef extern from "grpc/grpc_security.h":
+ # Declare this as an enum, this is the only way to make it a const in
+ # cython
+ enum: GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX
+
ctypedef enum grpc_ssl_roots_override_result:
GRPC_SSL_ROOTS_OVERRIDE_OK
GRPC_SSL_ROOTS_OVERRIDE_FAILED_PERMANENTLY
@@ -462,9 +466,12 @@
grpc_status_code status, const char *error_details)
ctypedef struct grpc_metadata_credentials_plugin:
- void (*get_metadata)(
+ int (*get_metadata)(
void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb, void *user_data)
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details)
void (*destroy)(void *state)
void *state
const char *type
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
index d860173..4f87261 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
@@ -171,14 +171,6 @@
gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME))
return <double>real_time.seconds + <double>real_time.nanoseconds / 1e9
- @staticmethod
- def infinite_future():
- return Timespec(float("+inf"))
-
- @staticmethod
- def infinite_past():
- return Timespec(float("-inf"))
-
def __richcmp__(Timespec self not None, Timespec other not None, int op):
cdef gpr_timespec self_c_time = self.c_time
cdef gpr_timespec other_c_time = other.c_time
@@ -454,7 +446,7 @@
self.i = self.i + 1
return result
else:
- raise StopIteration
+ raise StopIteration()
# TODO(https://github.com/grpc/grpc/issues/7950): Eliminate this; just use an
@@ -518,7 +510,7 @@
def __getitem__(self, size_t i):
if i >= self.c_metadata_array.count:
- raise IndexError
+ raise IndexError()
key = _slice_bytes(self.c_metadata_array.metadata[i].key)
value = _slice_bytes(self.c_metadata_array.metadata[i].value)
return Metadatum(key=key, value=value)
@@ -720,7 +712,7 @@
self.i = self.i + 1
return result
else:
- raise StopIteration
+ raise StopIteration()
cdef class Operations:
diff --git a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py
index 9f72b1f..6faab94 100644
--- a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py
+++ b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py
@@ -29,7 +29,7 @@
_REQUEST_SERIALIZER = lambda unused_request: _SERIALIZED_REQUEST
_REQUEST_DESERIALIZER = lambda unused_serialized_request: object()
_RESPONSE_SERIALIZER = lambda unused_response: _SERIALIZED_RESPONSE
-_RESPONSE_DESERIALIZER = lambda unused_serialized_resopnse: object()
+_RESPONSE_DESERIALIZER = lambda unused_serialized_response: object()
_SERVICE = 'test.TestService'
_UNARY_UNARY = 'UnaryUnary'
diff --git a/src/ruby/ext/grpc/rb_call_credentials.c b/src/ruby/ext/grpc/rb_call_credentials.c
index 049a869..4214a08 100644
--- a/src/ruby/ext/grpc/rb_call_credentials.c
+++ b/src/ruby/ext/grpc/rb_call_credentials.c
@@ -112,9 +112,12 @@
gpr_free(params);
}
-static void grpc_rb_call_credentials_plugin_get_metadata(
+static int grpc_rb_call_credentials_plugin_get_metadata(
void *state, grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb, void *user_data) {
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details) {
callback_params *params = gpr_malloc(sizeof(callback_params));
params->get_metadata = (VALUE)state;
params->context = context;
@@ -123,6 +126,7 @@
grpc_rb_event_queue_enqueue(grpc_rb_call_credentials_callback_with_gil,
(void *)(params));
+ return 0; // Async return.
}
static void grpc_rb_call_credentials_plugin_destroy(void *state) {
diff --git a/templates/test/cpp/naming/create_private_dns_zone.sh.template b/templates/test/cpp/naming/create_private_dns_zone.sh.template
new file mode 100644
index 0000000..14324b0
--- /dev/null
+++ b/templates/test/cpp/naming/create_private_dns_zone.sh.template
@@ -0,0 +1,4 @@
+%YAML 1.2
+--- |
+ <%namespace file="create_private_dns_zone_defs.include" import="*"/>\
+ ${create_private_dns_zone(resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)}
diff --git a/templates/test/cpp/naming/create_private_dns_zone_defs.include b/templates/test/cpp/naming/create_private_dns_zone_defs.include
new file mode 100644
index 0000000..465dd63
--- /dev/null
+++ b/templates/test/cpp/naming/create_private_dns_zone_defs.include
@@ -0,0 +1,32 @@
+<%def name="create_private_dns_zone(resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)">#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+gcloud alpha dns managed-zones create \\
+
+ ${resolver_gce_integration_tests_zone_id} \\
+
+ --dns-name=${resolver_tests_common_zone_name} \\
+
+ --description="GCE-DNS-private-zone-for-GRPC-testing" \\
+
+ --visibility=private \\
+
+ --networks=default</%def>
diff --git a/templates/test/cpp/naming/private_dns_zone_init.sh.template b/templates/test/cpp/naming/private_dns_zone_init.sh.template
new file mode 100644
index 0000000..d5ffd04
--- /dev/null
+++ b/templates/test/cpp/naming/private_dns_zone_init.sh.template
@@ -0,0 +1,4 @@
+%YAML 1.2
+--- |
+ <%namespace file="private_dns_zone_init_defs.include" import="*"/>\
+ ${private_dns_zone_init(all_integration_test_records, resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)}
diff --git a/templates/test/cpp/naming/private_dns_zone_init_defs.include b/templates/test/cpp/naming/private_dns_zone_init_defs.include
new file mode 100644
index 0000000..06bc8ad
--- /dev/null
+++ b/templates/test/cpp/naming/private_dns_zone_init_defs.include
@@ -0,0 +1,40 @@
+<%def name="private_dns_zone_init(records,resolver_gce_integration_tests_zone_id,resolver_tests_common_zone_name)">#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+gcloud dns record-sets transaction start -z=${resolver_gce_integration_tests_zone_id}
+
+% for r in records:
+gcloud dns record-sets transaction add \\
+
+ -z=${resolver_gce_integration_tests_zone_id} \\
+
+ --name=${r['name']}.${resolver_tests_common_zone_name} \\
+
+ --type=${r['type']} \\
+
+ --ttl=${r['ttl']} \\
+
+ ${r['data']}
+
+% endfor
+gcloud dns record-sets transaction describe -z=${resolver_gce_integration_tests_zone_id}
+gcloud dns record-sets transaction execute -z=${resolver_gce_integration_tests_zone_id}
+gcloud dns record-sets list -z=${resolver_gce_integration_tests_zone_id}</%def>
diff --git a/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include b/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include
new file mode 100644
index 0000000..2413ec5
--- /dev/null
+++ b/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include
@@ -0,0 +1,64 @@
+<%def name="resolver_gce_integration_tests(tests, records, resolver_tests_common_zone_name)">#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated
+
+set -ex
+
+if [[ "$GRPC_DNS_RESOLVER" == "" ]]; then
+ export GRPC_DNS_RESOLVER=ares
+elif [[ "$GRPC_DNS_RESOLVER" != ares ]]; then
+ echo "Unexpected: GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER. This test only works with c-ares resolver"
+ exit 1
+fi
+
+cd $(dirname $0)/../../..
+
+if [[ "$CONFIG" == "" ]]; then
+ export CONFIG=opt
+fi
+make resolver_component_test
+echo "Sanity check DNS records are resolveable with dig:"
+EXIT_CODE=0
+
+% for r in records:
+ONE_FAILED=0
+dig ${r['type']} ${r['name']}.${resolver_tests_common_zone_name} | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig ${r['type']} ${r['name']}.${resolver_tests_common_zone_name} FAILED"
+ exit 1
+fi
+
+% endfor
+echo "Sanity check PASSED. Run resolver tests:"
+
+% for test in tests:
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \\
+
+ --target_name='${test['target_name']}' \\
+
+ --expected_addrs='${test['expected_addrs']}' \\
+
+ --expected_chosen_service_config='${test['expected_chosen_service_config']}' \\
+
+ --expected_lb_policy='${test['expected_lb_policy']}' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: ${test['target_name']} FAILED"
+ EXIT_CODE=1
+fi
+
+% endfor
+exit $EXIT_CODE</%def>
diff --git a/templates/test/cpp/naming/resolver_gce_integration_tests_runner.sh.template b/templates/test/cpp/naming/resolver_gce_integration_tests_runner.sh.template
new file mode 100644
index 0000000..c728784
--- /dev/null
+++ b/templates/test/cpp/naming/resolver_gce_integration_tests_runner.sh.template
@@ -0,0 +1,4 @@
+%YAML 1.2
+--- |
+ <%namespace file="resolver_gce_integration_tests_defs.include" import="*"/>\
+ ${resolver_gce_integration_tests(resolver_gce_integration_test_cases, all_integration_test_records, resolver_tests_common_zone_name)}
diff --git a/templates/tools/dockerfile/python_deps.include b/templates/tools/dockerfile/python_deps.include
index 94b854a..bf1f57f 100644
--- a/templates/tools/dockerfile/python_deps.include
+++ b/templates/tools/dockerfile/python_deps.include
@@ -9,6 +9,6 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/test/core/security/credentials_test.c b/test/core/security/credentials_test.c
index 441c431..5ac5807 100644
--- a/test/core/security/credentials_test.c
+++ b/test/core/security/credentials_test.c
@@ -1036,39 +1036,46 @@
static const expected_md plugin_md[] = {{"foo", "bar"}, {"hi", "there"}};
-static void plugin_get_metadata_success(void *state,
- grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb,
- void *user_data) {
- size_t i;
- grpc_metadata md[GPR_ARRAY_SIZE(plugin_md)];
- plugin_state *s = (plugin_state *)state;
+static int plugin_get_metadata_success(
+ void *state, grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details) {
GPR_ASSERT(strcmp(context.service_url, test_service_url) == 0);
GPR_ASSERT(strcmp(context.method_name, test_method) == 0);
GPR_ASSERT(context.channel_auth_context == NULL);
GPR_ASSERT(context.reserved == NULL);
+ GPR_ASSERT(GPR_ARRAY_SIZE(plugin_md) <
+ GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX);
+ plugin_state *s = (plugin_state *)state;
*s = PLUGIN_GET_METADATA_CALLED_STATE;
- for (i = 0; i < GPR_ARRAY_SIZE(plugin_md); i++) {
- memset(&md[i], 0, sizeof(grpc_metadata));
- md[i].key = grpc_slice_from_copied_string(plugin_md[i].key);
- md[i].value = grpc_slice_from_copied_string(plugin_md[i].value);
+ for (size_t i = 0; i < GPR_ARRAY_SIZE(plugin_md); ++i) {
+ memset(&creds_md[i], 0, sizeof(grpc_metadata));
+ creds_md[i].key = grpc_slice_from_copied_string(plugin_md[i].key);
+ creds_md[i].value = grpc_slice_from_copied_string(plugin_md[i].value);
}
- cb(user_data, md, GPR_ARRAY_SIZE(md), GRPC_STATUS_OK, NULL);
+ *num_creds_md = GPR_ARRAY_SIZE(plugin_md);
+ return true; // Synchronous return.
}
static const char *plugin_error_details = "Could not get metadata for plugin.";
-static void plugin_get_metadata_failure(void *state,
- grpc_auth_metadata_context context,
- grpc_credentials_plugin_metadata_cb cb,
- void *user_data) {
- plugin_state *s = (plugin_state *)state;
+static int plugin_get_metadata_failure(
+ void *state, grpc_auth_metadata_context context,
+ grpc_credentials_plugin_metadata_cb cb, void *user_data,
+ grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX],
+ size_t *num_creds_md, grpc_status_code *status,
+ const char **error_details) {
GPR_ASSERT(strcmp(context.service_url, test_service_url) == 0);
GPR_ASSERT(strcmp(context.method_name, test_method) == 0);
GPR_ASSERT(context.channel_auth_context == NULL);
GPR_ASSERT(context.reserved == NULL);
+ plugin_state *s = (plugin_state *)state;
*s = PLUGIN_GET_METADATA_CALLED_STATE;
- cb(user_data, NULL, 0, GRPC_STATUS_UNAUTHENTICATED, plugin_error_details);
+ *status = GRPC_STATUS_UNAUTHENTICATED;
+ *error_details = gpr_strdup(plugin_error_details);
+ return true; // Synchronous return.
}
static void plugin_destroy(void *state) {
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index e54cd03..5dae5b0 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -1673,6 +1673,34 @@
kTestCredsPluginErrorMsg);
}
+TEST_P(SecureEnd2endTest, CompositeCallCreds) {
+ ResetStub();
+ EchoRequest request;
+ EchoResponse response;
+ ClientContext context;
+ const char kMetadataKey1[] = "call-creds-key1";
+ const char kMetadataKey2[] = "call-creds-key2";
+ const char kMetadataVal1[] = "call-creds-val1";
+ const char kMetadataVal2[] = "call-creds-val2";
+
+ context.set_credentials(CompositeCallCredentials(
+ MetadataCredentialsFromPlugin(std::unique_ptr<MetadataCredentialsPlugin>(
+ new TestMetadataCredentialsPlugin(kMetadataKey1, kMetadataVal1, true,
+ true))),
+ MetadataCredentialsFromPlugin(std::unique_ptr<MetadataCredentialsPlugin>(
+ new TestMetadataCredentialsPlugin(kMetadataKey2, kMetadataVal2, true,
+ true)))));
+ request.set_message("Hello");
+ request.mutable_param()->set_echo_metadata(true);
+
+ Status s = stub_->Echo(&context, request, &response);
+ EXPECT_TRUE(s.ok());
+ EXPECT_TRUE(MetadataContains(context.GetServerTrailingMetadata(),
+ kMetadataKey1, kMetadataVal1));
+ EXPECT_TRUE(MetadataContains(context.GetServerTrailingMetadata(),
+ kMetadataKey2, kMetadataVal2));
+}
+
TEST_P(SecureEnd2endTest, ClientAuthContext) {
ResetStub();
EchoRequest request;
diff --git a/test/cpp/naming/README.md b/test/cpp/naming/README.md
new file mode 100644
index 0000000..e331846
--- /dev/null
+++ b/test/cpp/naming/README.md
@@ -0,0 +1,43 @@
+# Resolver Tests
+
+This directory has tests and infrastructure for unit tests and GCE
+integration tests of gRPC resolver functionality.
+
+There are two different tests here:
+
+## Resolver unit tests (resolver "component" tests)
+
+These tests run per-change, along with the rest of the grpc unit tests.
+They query a local testing DNS server.
+
+## GCE integration tests
+
+These tests use the same test binary and the same test records
+as the unit tests, but they run against GCE DNS (this is done by
+running the test on a GCE instance and not specifying an authority
+in uris). These tests run in a background job, which needs to be
+actively monitored.
+
+## Making changes to test records
+
+After making a change to `resolver_test_record_groups.yaml`:
+
+1. Increment the "version number" in the `resolver_tests_common_zone_name`
+ DNS zone (this is a yaml field at the top
+ of `resolver_test_record_groups.yaml`).
+
+2. Regenerate projects.
+
+3. From the repo root, run:
+
+```
+$ test/cpp/naming/create_dns_private_zone.sh
+$ test/cpp/naming/private_dns_zone_init.sh
+```
+
+Note that these commands must be ran in environment that
+has access to the grpc-testing GCE project.
+
+If everything runs smoothly, then once the change is merged,
+the GCE DNS integration testing job will transition to the
+new records and continue passing.
diff --git a/test/cpp/naming/create_private_dns_zone.sh b/test/cpp/naming/create_private_dns_zone.sh
new file mode 100755
index 0000000..3d7520b
--- /dev/null
+++ b/test/cpp/naming/create_private_dns_zone.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+gcloud alpha dns managed-zones create \
+ resolver-tests-version-1-grpctestingexp-zone-id \
+ --dns-name=resolver-tests-version-1.grpctestingexp. \
+ --description="GCE-DNS-private-zone-for-GRPC-testing" \
+ --visibility=private \
+ --networks=default
diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py
index 3a51fef..9171815 100755
--- a/test/cpp/naming/gen_build_yaml.py
+++ b/test/cpp/naming/gen_build_yaml.py
@@ -24,6 +24,12 @@
_LOCAL_DNS_SERVER_ADDRESS = '127.0.0.1:15353'
+_TARGET_RECORDS_TO_SKIP_AGAINST_GCE = [
+ # TODO: enable this once able to upload the very large TXT record
+ # in this group to GCE DNS.
+ 'ipv4-config-causing-fallback-to-tcp',
+]
+
def _append_zone_name(name, zone_name):
return '%s.%s' % (name, zone_name)
@@ -33,21 +39,107 @@
out.append('%s,%s' % (addr['address'], str(addr['is_balancer'])))
return ';'.join(out)
+def _data_for_type(r_type, r_data, common_zone_name):
+ if r_type in ['A', 'AAAA']:
+ return ' '.join(map(lambda x: '\"%s\"' % x, r_data))
+ if r_type == 'SRV':
+ assert len(r_data) == 1
+ target = r_data[0].split(' ')[3]
+ uploadable_target = '%s.%s' % (target, common_zone_name)
+ uploadable = r_data[0].split(' ')
+ uploadable[3] = uploadable_target
+ return '\"%s\"' % ' '.join(uploadable)
+ if r_type == 'TXT':
+ assert len(r_data) == 1
+ chunks = []
+ all_data = r_data[0]
+ cur = 0
+ # Split TXT records that span more than 255 characters (the single
+ # string length-limit in DNS) into multiple strings. Each string
+ # needs to be wrapped with double-quotes, and all inner double-quotes
+ # are escaped. The wrapping double-quotes and inner backslashes can be
+ # counted towards the 255 character length limit (as observed with gcloud),
+ # so make sure all strings fit within that limit.
+ while len(all_data[cur:]) > 0:
+ next_chunk = '\"'
+ while len(next_chunk) < 254 and len(all_data[cur:]) > 0:
+ if all_data[cur] == '\"':
+ if len(next_chunk) < 253:
+ next_chunk += '\\\"'
+ else:
+ break
+ else:
+ next_chunk += all_data[cur]
+ cur += 1
+ next_chunk += '\"'
+ if len(next_chunk) > 255:
+ raise Exception('Bug: next chunk is too long.')
+ chunks.append(next_chunk)
+ # Wrap the whole record in single quotes to make sure all strings
+ # are associated with the same TXT record (to make it one bash token for
+ # gcloud)
+ return '\'%s\'' % ' '.join(chunks)
+
+# Convert DNS records from their "within a test group" format
+# of the yaml file to an easier form for the templates to use.
+def _gcloud_uploadable_form(test_cases, common_zone_name):
+ out = []
+ for group in test_cases:
+ if group['record_to_resolve'] in _TARGET_RECORDS_TO_SKIP_AGAINST_GCE:
+ continue
+ for record_name in group['records'].keys():
+ r_ttl = None
+ all_r_data = {}
+ for r_data in group['records'][record_name]:
+ # enforce records have the same TTL only for simplicity
+ if r_ttl is None:
+ r_ttl = r_data['TTL']
+ assert r_ttl == r_data['TTL'], '%s and %s differ' % (r_ttl, r_data['TTL'])
+ r_type = r_data['type']
+ if all_r_data.get(r_type) is None:
+ all_r_data[r_type] = []
+ all_r_data[r_type].append(r_data['data'])
+ for r_type in all_r_data.keys():
+ for r in out:
+ assert r['name'] != record_name or r['type'] != r_type, 'attempt to add a duplicate record'
+ out.append({
+ 'name': record_name,
+ 'ttl': r_ttl,
+ 'type': r_type,
+ 'data': _data_for_type(r_type, all_r_data[r_type], common_zone_name)
+ })
+ return out
+
+def _gce_dns_zone_id(resolver_component_data):
+ dns_name = resolver_component_data['resolver_tests_common_zone_name']
+ return dns_name.replace('.', '-') + 'zone-id'
+
+def _resolver_test_cases(resolver_component_data, records_to_skip):
+ out = []
+ for test_case in resolver_component_data['resolver_component_tests']:
+ if test_case['record_to_resolve'] in records_to_skip:
+ continue
+ out.append({
+ 'target_name': _append_zone_name(test_case['record_to_resolve'],
+ resolver_component_data['resolver_tests_common_zone_name']),
+ 'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']),
+ 'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''),
+ 'expected_lb_policy': (test_case['expected_lb_policy'] or ''),
+ })
+ return out
+
def main():
resolver_component_data = ''
with open('test/cpp/naming/resolver_test_record_groups.yaml') as f:
resolver_component_data = yaml.load(f)
json = {
- 'resolver_component_test_cases': [
- {
- 'target_name': _append_zone_name(test_case['record_to_resolve'],
- resolver_component_data['resolver_component_tests_common_zone_name']),
- 'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']),
- 'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''),
- 'expected_lb_policy': (test_case['expected_lb_policy'] or ''),
- } for test_case in resolver_component_data['resolver_component_tests']
- ],
+ 'resolver_tests_common_zone_name': resolver_component_data['resolver_tests_common_zone_name'],
+ 'resolver_gce_integration_tests_zone_id': _gce_dns_zone_id(resolver_component_data),
+ 'all_integration_test_records': _gcloud_uploadable_form(resolver_component_data['resolver_component_tests'],
+ resolver_component_data['resolver_tests_common_zone_name']),
+ 'resolver_gce_integration_test_cases': _resolver_test_cases(resolver_component_data, _TARGET_RECORDS_TO_SKIP_AGAINST_GCE),
+ 'resolver_component_test_cases': _resolver_test_cases(resolver_component_data, []),
'targets': [
{
'name': 'resolver_component_test' + unsecure_build_config_suffix,
diff --git a/test/cpp/naming/private_dns_zone_init.sh b/test/cpp/naming/private_dns_zone_init.sh
new file mode 100755
index 0000000..4eaf750
--- /dev/null
+++ b/test/cpp/naming/private_dns_zone_init.sh
@@ -0,0 +1,215 @@
+#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+gcloud dns record-sets transaction start -z=resolver-tests-version-1-grpctestingexp-zone-id
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=_grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. \
+ --type=SRV \
+ --ttl=2100 \
+ "0 0 1234 ipv4-single-target.resolver-tests-version-1.grpctestingexp."
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-single-target.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=_grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. \
+ --type=SRV \
+ --ttl=2100 \
+ "0 0 1234 ipv4-multi-target.resolver-tests-version-1.grpctestingexp."
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-multi-target.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.5" "1.2.3.6" "1.2.3.7"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=_grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. \
+ --type=SRV \
+ --ttl=2100 \
+ "0 0 1234 ipv6-single-target.resolver-tests-version-1.grpctestingexp."
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv6-single-target.resolver-tests-version-1.grpctestingexp. \
+ --type=AAAA \
+ --ttl=2100 \
+ "2607:f8b0:400a:801::1001"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=_grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. \
+ --type=SRV \
+ --ttl=2100 \
+ "0 0 1234 ipv6-multi-target.resolver-tests-version-1.grpctestingexp."
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv6-multi-target.resolver-tests-version-1.grpctestingexp. \
+ --type=AAAA \
+ --ttl=2100 \
+ "2607:f8b0:400a:801::1002" "2607:f8b0:400a:801::1003" "2607:f8b0:400a:801::1004"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \
+ --type=TXT \
+ --ttl=2100 \
+ '"grpc_config=[{\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"SimpleService\",\"waitForReady\":true}]}]}}]"'
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=_grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \
+ --type=SRV \
+ --ttl=2100 \
+ "0 0 1234 ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp."
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. \
+ --type=TXT \
+ --ttl=2100 \
+ '"grpc_config=[{\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"NoSrvSimpleService\",\"waitForReady\":true}]}]}}]"'
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. \
+ --type=TXT \
+ --ttl=2100 \
+ '"grpc_config=[{\"clientLanguage\":[\"python\"],\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"PythonService\",\"waitForReady\":true}]}]}}]"'
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. \
+ --type=TXT \
+ --ttl=2100 \
+ '"grpc_config=[{\"percentage\":0,\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"CppService\",\"waitForReady\":true}]}]}}]"'
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. \
+ --type=TXT \
+ --ttl=2100 \
+ '"grpc_config=[{\"clientLanguage\":[\"go\"],\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"GoService\",\"waitForReady\":true}]}]}},{\"clientLanguage\":[\"c++\"],\"serviceConfig\":{" "\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"CppService\",\"waitForReady\":true}]}]}}]"'
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. \
+ --type=TXT \
+ --ttl=2100 \
+ '"grpc_config=[{\"percentage\":0,\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"NeverPickedService\",\"waitForReady\":true}]}]}},{\"percentage\":100,\"serviceConfig\":{\"loadBalanc" "ingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"AlwaysPickedService\",\"waitForReady\":true}]}]}}]"'
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=_grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
+ --type=SRV \
+ --ttl=2100 \
+ "0 0 1234 balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp."
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
+ --type=A \
+ --ttl=2100 \
+ "1.2.3.4"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=_grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
+ --type=SRV \
+ --ttl=2100 \
+ "0 0 1234 balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp."
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
+ --type=AAAA \
+ --ttl=2100 \
+ "2607:f8b0:400a:801::1002"
+
+gcloud dns record-sets transaction add \
+ -z=resolver-tests-version-1-grpctestingexp-zone-id \
+ --name=srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
+ --type=AAAA \
+ --ttl=2100 \
+ "2607:f8b0:400a:801::1002"
+
+gcloud dns record-sets transaction describe -z=resolver-tests-version-1-grpctestingexp-zone-id
+gcloud dns record-sets transaction execute -z=resolver-tests-version-1-grpctestingexp-zone-id
+gcloud dns record-sets list -z=resolver-tests-version-1-grpctestingexp-zone-id
diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh
index cf71c9d..407db5e 100755
--- a/test/cpp/naming/resolver_component_tests_runner.sh
+++ b/test/cpp/naming/resolver_component_tests_runner.sh
@@ -73,7 +73,7 @@
# in the resolver.
$FLAGS_test_bin_path \
- --target_name='srv-ipv4-single-target.resolver-tests.grpctestingexp.' \
+ --target_name='srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@@ -81,7 +81,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='srv-ipv4-multi-target.resolver-tests.grpctestingexp.' \
+ --target_name='srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@@ -89,7 +89,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='srv-ipv6-single-target.resolver-tests.grpctestingexp.' \
+ --target_name='srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@@ -97,7 +97,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='srv-ipv6-multi-target.resolver-tests.grpctestingexp.' \
+ --target_name='srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@@ -105,7 +105,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='srv-ipv4-simple-service-config.resolver-tests.grpctestingexp.' \
+ --target_name='srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
@@ -113,7 +113,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='ipv4-no-srv-simple-service-config.resolver-tests.grpctestingexp.' \
+ --target_name='ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
@@ -121,7 +121,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='ipv4-no-config-for-cpp.resolver-tests.grpctestingexp.' \
+ --target_name='ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@@ -129,7 +129,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests.grpctestingexp.' \
+ --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@@ -137,7 +137,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='ipv4-second-language-is-cpp.resolver-tests.grpctestingexp.' \
+ --target_name='ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
@@ -145,7 +145,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='ipv4-config-with-percentages.resolver-tests.grpctestingexp.' \
+ --target_name='ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
@@ -153,7 +153,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \
+ --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@@ -161,7 +161,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \
+ --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@@ -169,7 +169,7 @@
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
- --target_name='ipv4-config-causing-fallback-to-tcp.resolver-tests.grpctestingexp.' \
+ --target_name='ipv4-config-causing-fallback-to-tcp.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='' \
diff --git a/test/cpp/naming/resolver_gce_integration_tests_runner.sh b/test/cpp/naming/resolver_gce_integration_tests_runner.sh
new file mode 100755
index 0000000..b20d18d
--- /dev/null
+++ b/test/cpp/naming/resolver_gce_integration_tests_runner.sh
@@ -0,0 +1,359 @@
+#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated
+
+set -ex
+
+if [[ "$GRPC_DNS_RESOLVER" == "" ]]; then
+ export GRPC_DNS_RESOLVER=ares
+elif [[ "$GRPC_DNS_RESOLVER" != ares ]]; then
+ echo "Unexpected: GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER. This test only works with c-ares resolver"
+ exit 1
+fi
+
+cd $(dirname $0)/../../..
+
+if [[ "$CONFIG" == "" ]]; then
+ export CONFIG=opt
+fi
+make resolver_component_test
+echo "Sanity check DNS records are resolveable with dig:"
+EXIT_CODE=0
+
+ONE_FAILED=0
+dig SRV _grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A ipv4-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig SRV _grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A ipv4-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig SRV _grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig AAAA ipv6-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig AAAA ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig SRV _grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig AAAA ipv6-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig AAAA ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig TXT srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig TXT srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig SRV _grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig TXT ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig TXT ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig TXT ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig TXT ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig TXT ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig TXT ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig TXT ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig TXT ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig TXT ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig TXT ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig SRV _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig A srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig A srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig SRV _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig AAAA balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig AAAA balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+ONE_FAILED=0
+dig AAAA srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Sanity check: dig AAAA srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
+ exit 1
+fi
+
+echo "Sanity check PASSED. Run resolver tests:"
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.4:1234,True' \
+ --expected_chosen_service_config='' \
+ --expected_lb_policy='' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \
+ --expected_chosen_service_config='' \
+ --expected_lb_policy='' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \
+ --expected_chosen_service_config='' \
+ --expected_lb_policy='' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \
+ --expected_chosen_service_config='' \
+ --expected_lb_policy='' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.4:1234,True' \
+ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \
+ --expected_lb_policy='round_robin' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.4:443,False' \
+ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \
+ --expected_lb_policy='round_robin' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.4:443,False' \
+ --expected_chosen_service_config='' \
+ --expected_lb_policy='' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.4:443,False' \
+ --expected_chosen_service_config='' \
+ --expected_lb_policy='' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.4:443,False' \
+ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \
+ --expected_lb_policy='round_robin' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.4:443,False' \
+ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \
+ --expected_lb_policy='round_robin' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \
+ --expected_chosen_service_config='' \
+ --expected_lb_policy='' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+ONE_FAILED=0
+bins/$CONFIG/resolver_component_test \
+ --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \
+ --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \
+ --expected_chosen_service_config='' \
+ --expected_lb_policy='' || ONE_FAILED=1
+if [[ "$ONE_FAILED" != 0 ]]; then
+ echo "Test based on target record: srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
+ EXIT_CODE=1
+fi
+
+exit $EXIT_CODE
diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml
index 33d774c..2b32043 100644
--- a/test/cpp/naming/resolver_test_record_groups.yaml
+++ b/test/cpp/naming/resolver_test_record_groups.yaml
@@ -1,4 +1,4 @@
-resolver_component_tests_common_zone_name: resolver-tests.grpctestingexp.
+resolver_tests_common_zone_name: resolver-tests-version-1.grpctestingexp.
resolver_component_tests:
- expected_addrs:
- {address: '1.2.3.4:1234', is_balancer: true}
diff --git a/test/cpp/naming/test_dns_server.py b/test/cpp/naming/test_dns_server.py
index 9d4b89c..9f42f65 100755
--- a/test/cpp/naming/test_dns_server.py
+++ b/test/cpp/naming/test_dns_server.py
@@ -66,7 +66,7 @@
with open(args.records_config_path) as config:
test_records_config = yaml.load(config)
- common_zone_name = test_records_config['resolver_component_tests_common_zone_name']
+ common_zone_name = test_records_config['resolver_tests_common_zone_name']
for group in test_records_config['resolver_component_tests']:
for name in group['records'].keys():
for record in group['records'][name]:
diff --git a/tools/distrib/pull_requests_interval.sh b/tools/distrib/pull_requests_interval.sh
new file mode 100755
index 0000000..7a6c702
--- /dev/null
+++ b/tools/distrib/pull_requests_interval.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$1" = "x" ] ; then
+ echo "Usage: $0 <first ref> [second ref]"
+ exit 1
+else
+ first=$1
+fi
+
+if [ -n $2 ] ; then
+ second=HEAD
+fi
+
+if [ -e ~/github-credentials.vars ] ; then
+ . ~/github-credentials.vars
+fi
+
+if [ "x$github_client_id" = "x" ] || [ "x$github_client_secret" = "x" ] ; then
+ echo "Warning: you don't have github credentials set."
+ echo
+ echo "You may end up exceeding guest quota quickly."
+ echo "You can create an application for yourself,"
+ echo "and get its credentials. Go to"
+ echo
+ echo " https://github.com/settings/developers"
+ echo
+ echo "and click 'Register a new application'."
+ echo
+ echo "From the application's information, copy/paste"
+ echo "its Client ID and Client Secret, into the file"
+ echo
+ echo " ~/github-credentials.vars"
+ echo
+ echo "with the following format:"
+ echo
+ echo "github_client_id=0123456789abcdef0123"
+ echo "github_client_secret=0123456789abcdef0123456789abcdef"
+ echo
+ echo
+ addendum=""
+else
+ addendum="?client_id=$github_client_id&client_secret=$github_client_secret"
+fi
+
+unset notfirst
+echo "["
+git log --pretty=oneline $1..$2 |
+ grep '[^ ]\+ Merge pull request #[0-9]\{4,6\} ' |
+ cut -f 2 -d# |
+ cut -f 1 -d\ |
+ sort -u |
+ while read id ; do
+ if [ "x$notfirst" = "x" ] ; then
+ notfirst=true
+ else
+ echo ","
+ fi
+ echo -n " {\"url\": \"https://github.com/grpc/grpc/pull/$id\","
+ out=`mktemp`
+ curl -s "https://api.github.com/repos/grpc/grpc/pulls/$id$addendum" > $out
+ echo -n " "`grep '"title"' $out`
+ echo -n " "`grep '"login"' $out | head -1`
+ echo -n " \"pr\": $id }"
+ rm $out
+ done
+echo
+echo "]"
diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh
index 3c9235b..7175f1e 100755
--- a/tools/distrib/pylint_code.sh
+++ b/tools/distrib/pylint_code.sh
@@ -29,7 +29,7 @@
virtualenv $VIRTUALENV
PYTHON=$(realpath $VIRTUALENV/bin/python)
-$PYTHON -m pip install --upgrade pip
+$PYTHON -m pip install --upgrade pip==9.0.1
$PYTHON -m pip install pylint==1.6.5
for dir in "${DIRS[@]}"; do
diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py
index 6f6d43c..1822e51 100755
--- a/tools/distrib/python/docgen.py
+++ b/tools/distrib/python/docgen.py
@@ -60,7 +60,7 @@
subprocess_arguments_list = [
{'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment},
- {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip'],
+ {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip==9.0.1'],
'env': environment},
{'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH],
'env': environment},
diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh
index dbb6b5c..e5beb70 100755
--- a/tools/distrib/yapf_code.sh
+++ b/tools/distrib/yapf_code.sh
@@ -33,7 +33,7 @@
virtualenv $VIRTUALENV
PYTHON=$(realpath "${VIRTUALENV}/bin/python")
-$PYTHON -m pip install --upgrade pip
+$PYTHON -m pip install --upgrade pip==9.0.1
$PYTHON -m pip install --upgrade futures
$PYTHON -m pip install yapf==0.16.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile
index dbf5802..ea82476 100644
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile
@@ -60,7 +60,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile
index 4ccfbc4..56b8be8 100644
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile
@@ -59,7 +59,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile
index 7cfe98c..38d377c 100644
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile
@@ -60,7 +60,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile
index febe2fa..73c41a4 100644
--- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile
@@ -28,7 +28,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile
index 3a516cb..7c083de 100644
--- a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile
@@ -28,7 +28,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile
index acb640a..61efc18 100644
--- a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile
@@ -28,7 +28,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
index 354b7bf..278b09a 100644
--- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
@@ -28,7 +28,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile
index 92a542f..d566324 100644
--- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile
@@ -43,7 +43,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile
index 92a542f..d566324 100644
--- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile
@@ -43,7 +43,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile
index 4343d56..f4c3e41 100644
--- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile
@@ -60,7 +60,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile
index 271c6e7..d165307 100644
--- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile
@@ -60,7 +60,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile
index 7bcada6..2217b10 100644
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile
@@ -60,7 +60,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
index 40d46fc..3e31e67 100644
--- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile
index 1ae50c1..af5e7d6 100644
--- a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile
@@ -36,7 +36,7 @@
zip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
index 888a37b..3492dd7 100644
--- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
index 319f1e1..f8cbf35 100644
--- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
+++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
index 61f005d..6966d6b 100644
--- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
index f35247e..016034a 100644
--- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile
index ce1badf..50104ad 100644
--- a/tools/dockerfile/test/fuzzer/Dockerfile
+++ b/tools/dockerfile/test/fuzzer/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
index 59fe4d8..1a4b681 100644
--- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
@@ -121,7 +121,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile
index 103be84..4f18dba 100644
--- a/tools/dockerfile/test/node_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile
@@ -75,7 +75,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile
index f6d426b..1399502 100644
--- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile
@@ -75,7 +75,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile
index ae82a8d..56dc604 100644
--- a/tools/dockerfile/test/php_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/python_alpine_x64/Dockerfile b/tools/dockerfile/test/python_alpine_x64/Dockerfile
index 7bd11d7..7584ab8 100644
--- a/tools/dockerfile/test/python_alpine_x64/Dockerfile
+++ b/tools/dockerfile/test/python_alpine_x64/Dockerfile
@@ -36,7 +36,7 @@
zip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile
index d5d781c..8d89f50 100644
--- a/tools/dockerfile/test/python_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile
index 3b4ad12..f8cbdc5 100644
--- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile
+++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
index 3d879bb..5d7f80b 100644
--- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index dff979d..487ce15 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -64,7 +64,7 @@
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh
index 88d8de7..8d900f1 100755
--- a/tools/gce/linux_performance_worker_init.sh
+++ b/tools/gce/linux_performance_worker_init.sh
@@ -72,6 +72,7 @@
sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang
# Python dependencies
+sudo pip install --upgrade pip==9.0.1
sudo pip install tabulate
sudo pip install google-api-python-client
sudo pip install virtualenv
diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.sh b/tools/internal_ci/linux/grpc_performance_profile_daily.sh
index 25523e2..34d41bc 100755
--- a/tools/internal_ci/linux/grpc_performance_profile_daily.sh
+++ b/tools/internal_ci/linux/grpc_performance_profile_daily.sh
@@ -22,6 +22,8 @@
CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'`
+./tools/run_tests/start_port_server.py || true
+
make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS
bins/opt/memory_profile_test
bq load microbenchmarks.memory memory_usage.csv
diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md
index f92dc69..c2f3543 100644
--- a/tools/interop_matrix/README.md
+++ b/tools/interop_matrix/README.md
@@ -5,6 +5,21 @@
The setup builds gRPC docker images for each language/runtime and upload it to Google Container Registry (GCR). These images, encapsulating gRPC stack
from specific releases/tag, are used to test version compatiblity between gRPC release versions.
+## Step-by-step instructions for adding a new release to compatibility test
+We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are:
+- Add (or update) an entry in ./client_matrix.py file to reference the github tag for the release.
+- Build new client docker image(s). For example, for java release `v1.9.9`, do
+ - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language=java`
+- Verify that the new docker image was built successfully and uploaded to GCR. For example,
+ - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8`
+ - should show an image entry with tag `v1.9.9`.
+- Verify the just-created docker client image would pass backward compatibility test (it should). For example,
+ - `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9` followed by
+ - `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 ./testcases/java__master`
+- git commit the change and merge it to upstream/master.
+- (Optional) clean up the tmp directory to where grpc source is cloned at `/export/hda3/tmp/grpc_matrix/`.
+For more details on each step, refer to sections below.
+
## Instructions for creating GCR images
- Edit `./client_matrix.py` to include desired gRPC release.
- Run `tools/interop_matrix/create_matrix_images.py`. Useful options:
@@ -45,3 +60,4 @@
Note:
- File path starting with `tools/` or `template/` are relative to the grpc repo root dir. File path starting with `./` are relative to current directory (`tools/interop_matrix`).
+- Creating and referencing images in GCR require read and write permission to Google Container Registry path gcr.io/grpc-testing.
diff --git a/tools/jenkins/run_performance_profile_daily.sh b/tools/jenkins/run_performance_profile_daily.sh
index 04a2464..48d82a9 100755
--- a/tools/jenkins/run_performance_profile_daily.sh
+++ b/tools/jenkins/run_performance_profile_daily.sh
@@ -29,4 +29,6 @@
BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
+./tools/run_tests/start_port_server.py || true
+
$PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency -b $BENCHMARKS_TO_RUN
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py
index 5aa11ac..516d110 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py
@@ -23,6 +23,7 @@
import sys
import os
+import random
import argparse
import multiprocessing
import subprocess
@@ -32,6 +33,12 @@
os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
import comment_on_pr
+sys.path.append(
+ os.path.join(
+ os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
+ 'python_utils'))
+import jobset
+
def _args():
argp = argparse.ArgumentParser(
@@ -125,8 +132,13 @@
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
- bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
- bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
+ jobs_list = []
+ jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, args.regex, args.counters)
+ jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, args.regex, args.counters)
+
+ # shuffle all jobs to eliminate noise from GCE CPU drift
+ random.shuffle(jobs_list, random.SystemRandom().random)
+ jobset.run(jobs_list, maxjobs=args.jobs)
diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, args.track, old,
'new', args.counters)
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py
index 206f7c5..81db5a2 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_run.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py
@@ -95,11 +95,12 @@
shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1,
loops),
verbose_success=True,
+ cpu_cost=2,
timeout_seconds=60 * 60)) # one hour
return jobs_list
-def run(name, benchmarks, jobs, loops, regex, counters):
+def create_jobs(name, benchmarks, loops, regex, counters):
jobs_list = []
for loop in range(0, loops):
for bm in benchmarks:
@@ -108,9 +109,11 @@
jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop,
loops)
random.shuffle(jobs_list, random.SystemRandom().random)
- jobset.run(jobs_list, maxjobs=jobs)
+ return jobs_list
if __name__ == '__main__':
args = _args()
- run(args.name, args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
+ jobs_list = create_jobs(args.name, args.benchmarks, args.loops,
+ args.regex, args.counters)
+ jobset.run(jobs_list, maxjobs=args.jobs)
diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh
index be65055..e362082 100755
--- a/tools/run_tests/helper_scripts/build_python.sh
+++ b/tools/run_tests/helper_scripts/build_python.sh
@@ -152,7 +152,7 @@
cd $PWD
}
-$VENV_PYTHON -m pip install --upgrade pip
+$VENV_PYTHON -m pip install --upgrade pip==9.0.1
$VENV_PYTHON -m pip install setuptools
$VENV_PYTHON -m pip install cython
$VENV_PYTHON -m pip install six enum34 protobuf futures
diff --git a/tools/run_tests/helper_scripts/run_grpc-node.sh b/tools/run_tests/helper_scripts/run_grpc-node.sh
new file mode 100755
index 0000000..25f149f
--- /dev/null
+++ b/tools/run_tests/helper_scripts/run_grpc-node.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script runs grpc/grpc-node tests with their grpc submodule updated
+# to this reference
+
+# cd to gRPC root directory
+cd $(dirname $0)/../../..
+
+CURRENT_COMMIT=$(git rev-parse --verify HEAD)
+
+rm -rf ./../grpc-node
+git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node
+cd ./../grpc-node
+
+./test-grpc-submodule.sh $CURRENT_COMMIT
diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py
index f99143f..393aef8 100644
--- a/tools/run_tests/python_utils/filter_pull_request_tests.py
+++ b/tools/run_tests/python_utils/filter_pull_request_tests.py
@@ -47,7 +47,7 @@
_CORE_TEST_SUITE = TestSuite(['c'])
_CPP_TEST_SUITE = TestSuite(['c++'])
_CSHARP_TEST_SUITE = TestSuite(['csharp'])
-_NODE_TEST_SUITE = TestSuite(['node'])
+_NODE_TEST_SUITE = TestSuite(['node', 'grpc-node'])
_OBJC_TEST_SUITE = TestSuite(['objc'])
_PHP_TEST_SUITE = TestSuite(['php', 'php7'])
_PYTHON_TEST_SUITE = TestSuite(['python'])
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 6697149..6e2115c 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -444,6 +444,67 @@
return self.make_target
+# This tests Node on grpc/grpc-node and will become the standard for Node testing
+class RemoteNodeLanguage(object):
+
+ def __init__(self):
+ self.platform = platform_string()
+
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ # Note: electron ABI only depends on major and minor version, so that's all
+ # we should specify in the compiler argument
+ _check_compiler(self.args.compiler, ['default', 'node0.12',
+ 'node4', 'node5', 'node6',
+ 'node7', 'node8',
+ 'electron1.3', 'electron1.6'])
+ if self.args.compiler == 'default':
+ self.runtime = 'node'
+ self.node_version = '8'
+ else:
+ if self.args.compiler.startswith('electron'):
+ self.runtime = 'electron'
+ self.node_version = self.args.compiler[8:]
+ else:
+ self.runtime = 'node'
+ # Take off the word "node"
+ self.node_version = self.args.compiler[4:]
+
+ # TODO: update with Windows/electron scripts when available for grpc/grpc-node
+ def test_specs(self):
+ if self.platform == 'windows':
+ return [self.config.job_spec(['tools\\run_tests\\helper_scripts\\run_node.bat'])]
+ else:
+ return [self.config.job_spec(['tools/run_tests/helper_scripts/run_grpc-node.sh'],
+ None,
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+
+ def pre_build_steps(self):
+ return []
+
+ def make_targets(self):
+ return []
+
+ def make_options(self):
+ return []
+
+ def build_steps(self):
+ return []
+
+ def post_tests_steps(self):
+ return []
+
+ def makefile_name(self):
+ return 'Makefile'
+
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)
+
+ def __str__(self):
+ return 'grpc-node'
+
+
class NodeLanguage(object):
def __init__(self):
@@ -1067,6 +1128,7 @@
_LANGUAGES = {
'c++': CLanguage('cxx', 'c++'),
'c': CLanguage('c', 'c'),
+ 'grpc-node': RemoteNodeLanguage(),
'node': NodeLanguage(),
'node_express': NodeExpressLanguage(),
'php': PhpLanguage(),
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 2417c5a..34d839e 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -169,7 +169,7 @@
inner_jobs=inner_jobs,
timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- test_jobs += _generate_jobs(languages=['ruby', 'php'],
+ test_jobs += _generate_jobs(languages=['grpc-node', 'ruby', 'php'],
configs=['dbg', 'opt'],
platforms=['linux', 'macos'],
labels=['basictests', 'multilang'],
diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py
index bea8125..3ebb938 100755
--- a/tools/run_tests/sanity/check_test_filtering.py
+++ b/tools/run_tests/sanity/check_test_filtering.py
@@ -25,7 +25,7 @@
from run_tests_matrix import _create_test_jobs, _create_portability_test_jobs
import python_utils.filter_pull_request_tests as filter_pull_request_tests
-_LIST_OF_LANGUAGE_LABELS = ['c', 'c++', 'csharp', 'node', 'objc', 'php', 'php7', 'python', 'ruby']
+_LIST_OF_LANGUAGE_LABELS = ['c', 'c++', 'csharp', 'grpc-node', 'node', 'objc', 'php', 'php7', 'python', 'ruby']
_LIST_OF_PLATFORM_LABELS = ['linux', 'macos', 'windows']
class TestFilteringTest(unittest.TestCase):