Merge pull request #5328 from murgatroid99/node_jwt_interop_fix

Node: fix metadata validation bug, improve error reporting
diff --git a/Makefile b/Makefile
index 6c7febd..62e873d 100644
--- a/Makefile
+++ b/Makefile
@@ -3162,6 +3162,7 @@
     test/cpp/util/create_test_channel.cc \
     test/cpp/util/string_ref_helper.cc \
     test/cpp/util/subprocess.cc \
+    test/cpp/util/test_credentials_provider.cc \
 
 
 LIBGRPC++_TEST_UTIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_TEST_UTIL_SRC))))
@@ -3212,6 +3213,7 @@
 $(OBJDIR)/$(CONFIG)/test/cpp/util/create_test_channel.o: $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.pb.cc $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.cc
 $(OBJDIR)/$(CONFIG)/test/cpp/util/string_ref_helper.o: $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.pb.cc $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.cc
 $(OBJDIR)/$(CONFIG)/test/cpp/util/subprocess.o: $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.pb.cc $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.cc
+$(OBJDIR)/$(CONFIG)/test/cpp/util/test_credentials_provider.o: $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.pb.cc $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.cc
 
 
 LIBGRPC++_UNSECURE_SRC = \
@@ -13015,6 +13017,7 @@
 test/cpp/util/string_ref_helper.cc: $(OPENSSL_DEP)
 test/cpp/util/subprocess.cc: $(OPENSSL_DEP)
 test/cpp/util/test_config.cc: $(OPENSSL_DEP)
+test/cpp/util/test_credentials_provider.cc: $(OPENSSL_DEP)
 endif
 
 .PHONY: all strip tools dep_error openssl_dep_error openssl_dep_message git_update stop buildtests buildtests_c buildtests_cxx test test_c test_cxx install install_c install_cxx install-headers install-headers_c install-headers_cxx install-shared install-shared_c install-shared_cxx install-static install-static_c install-static_cxx strip strip-shared strip-static strip_c strip-shared_c strip-static_c strip_cxx strip-shared_cxx strip-static_cxx dep_c dep_cxx bins_dep_c bins_dep_cxx clean
diff --git a/build.yaml b/build.yaml
index b639b5d..5b1878e 100644
--- a/build.yaml
+++ b/build.yaml
@@ -716,6 +716,7 @@
   - test/cpp/util/create_test_channel.h
   - test/cpp/util/string_ref_helper.h
   - test/cpp/util/subprocess.h
+  - test/cpp/util/test_credentials_provider.h
   src:
   - src/proto/grpc/testing/echo_messages.proto
   - src/proto/grpc/testing/echo.proto
@@ -726,6 +727,7 @@
   - test/cpp/util/create_test_channel.cc
   - test/cpp/util/string_ref_helper.cc
   - test/cpp/util/subprocess.cc
+  - test/cpp/util/test_credentials_provider.cc
   deps:
   - grpc++
   - grpc_test_util
diff --git a/doc/load-balancing.md b/doc/load-balancing.md
new file mode 100644
index 0000000..681be02
--- /dev/null
+++ b/doc/load-balancing.md
@@ -0,0 +1,97 @@
+Load Balancing in gRPC
+=======================
+
+# Objective
+
+To design a load balancing API between a gRPC client and a Load Balancer to
+instruct the client how to send load to multiple backend servers. 
+
+# Background
+
+Prior to any gRPC specifics, we explore some usual ways to approach load
+balancing.
+
+### Proxy Model
+
+Using a proxy provides a solid trustable client that can report load to the load
+balancing system. Proxies typically require more resources to operate since they
+have temporary copies of the RPC request and response. This model also increases
+latency to the RPCs.
+
+The proxy model was deemed inefficient when considering request heavy services
+like storage. 
+
+### Balancing-aware Client
+
+This thicker client places more of the load balancing logic in the client. For
+example, the client could contain many load balancing policies (Round Robin,
+Random, etc) used to select servers from a list. In this model, a list of
+servers would be either statically configured in the client, provided by the
+name resolution system, an external load balancer, etc. In any case, the client
+is responsible for choosing the preferred server from the list. 
+
+One of the drawbacks of this approach is writing and maintaining the load
+balancing policies in multiple languages and/or versions of the clients. These
+policies can be fairly complicated. Some of the algorithms also require client
+to server communication so the client would need to get thicker to support
+additional RPCs to get health or load information in addition to sending RPCs
+for user requests.
+
+It would also significantly complicate the client's code: the new design hides
+the load balancing complexity of multiple layers and presents it as a simple
+list of servers to the client.
+
+### External Load Balancing Service
+
+The client load balancing code is kept simple and portable, implementing
+well-known algorithms (ie, Round Robin) for server selection.
+Complex load balancing algorithms are instead provided by the load balancer. The
+client relies on the load balancer to provide _load balancing configuration_ and
+_the list of servers_ to which the client should send requests. The balancer
+updates the server list as needed to balance the load as well as handle server
+unavailability or health issues. The load balancer will make any necessary
+complex decisions and inform the client. The load balancer may communicate with
+the backend servers to collect load and health information.
+
+# Proposed Architecture
+
+The gRPC load balancing approach follows the third approach, by having an
+external load balancer which provides simple clients with a list of servers.
+
+## Client
+
+When establishing a gRPC stream to the balancer, the client will send an initial
+request to the load balancer (via a regular gRPC message). The load balancer
+will respond with client config (including, for example, settings for flow
+control, RPC deadlines, etc.) or a redirect to another load balancer. If the
+balancer did not redirect the client, it will then send a list of servers to the
+client. The client will contain simple load balancing logic for choosing the
+next server when it needs to send a request.
+
+## Load Balancer
+
+The Load Balancer is responsible for providing the client with a list of servers
+and client RPC parameters. The balancer chooses when to update the list of
+servers and can decide whether to provide a complete list, a subset, or a
+specific list of “picked” servers in a particular order. The balancer can
+optionally provide an expiration interval after which the server list should no
+longer be trusted and should be updated by the balancer.
+
+The load balancer may open reporting streams to each server contained in the
+server list. These streams are primarily used for load reporting. For example,
+Weighted Round Robin requires that the servers report utilization to the load
+balancer in order to compute the next list of servers.
+
+## Server
+
+The gRPC Server is responsible for answering RPC requests and providing
+responses to the client. The server will also report load to the load balancer
+if a reporting stream was opened for this purpose.
+
+### Security 
+
+The load balancer may be separate from the actual server backends and a
+compromise of the load balancer should only lead to a compromise of the
+loadbalancing functionality. In other words, a compromised load balancer should
+not be able to cause a client to trust a (potentially malicious) backend server
+any more than in a comparable situation without loadbalancing. 
diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h
index e65349d..5e450b0 100644
--- a/include/grpc++/impl/codegen/call.h
+++ b/include/grpc++/impl/codegen/call.h
@@ -272,6 +272,7 @@
 class DeserializeFunc {
  public:
   virtual Status Deserialize(grpc_byte_buffer* buf, int max_message_size) = 0;
+  virtual ~DeserializeFunc() {}
 };
 
 template <class R>
@@ -283,6 +284,8 @@
     return SerializationTraits<R>::Deserialize(buf, message_, max_message_size);
   }
 
+  ~DeserializeFuncType() override {}
+
  private:
   R* message_;  // Not a managed pointer because management is external to this
 };
diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h
index b34bb09..5868b3f 100644
--- a/src/core/iomgr/pollset_posix.h
+++ b/src/core/iomgr/pollset_posix.h
@@ -142,6 +142,10 @@
 void grpc_remove_fd_from_all_epoll_sets(int fd);
 
 /* override to allow tests to hook poll() usage */
+/* NOTE: Any changes to grpc_poll_function must take place when the gRPC
+   is certainly not doing any polling anywhere.
+   Otherwise, there might be a race between changing the variable and actually
+   doing a polling operation */
 typedef int (*grpc_poll_function_type)(struct pollfd *, nfds_t, int);
 extern grpc_poll_function_type grpc_poll_function;
 extern grpc_wakeup_fd grpc_global_wakeup_fd;
diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc
index a194c61..a15cbd7 100644
--- a/test/cpp/end2end/async_end2end_test.cc
+++ b/test/cpp/end2end/async_end2end_test.cc
@@ -43,6 +43,7 @@
 #include <grpc/grpc.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
+#include <grpc/support/tls.h>
 #include <gtest/gtest.h>
 
 #include "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h"
@@ -59,6 +60,8 @@
 using grpc::testing::EchoResponse;
 using std::chrono::system_clock;
 
+GPR_TLS_DECL(g_is_async_end2end_test);
+
 namespace grpc {
 namespace testing {
 
@@ -67,9 +70,11 @@
 void* tag(int i) { return (void*)(intptr_t)i; }
 
 #ifdef GPR_POSIX_SOCKET
-static int assert_non_blocking_poll(struct pollfd* pfds, nfds_t nfds,
-                                    int timeout) {
-  GPR_ASSERT(timeout == 0);
+static int maybe_assert_non_blocking_poll(struct pollfd* pfds, nfds_t nfds,
+                                          int timeout) {
+  if (gpr_tls_get(&g_is_async_end2end_test)) {
+    GPR_ASSERT(timeout == 0);
+  }
   return poll(pfds, nfds, timeout);
 }
 
@@ -86,21 +91,21 @@
   grpc_poll_function_type prev_;
 };
 
-class PollingCheckRegion : public PollOverride {
+class PollingOverrider : public PollOverride {
  public:
-  explicit PollingCheckRegion(bool allow_blocking)
-      : PollOverride(allow_blocking ? poll : assert_non_blocking_poll) {}
+  explicit PollingOverrider(bool allow_blocking)
+      : PollOverride(allow_blocking ? poll : maybe_assert_non_blocking_poll) {}
 };
 #else
-class PollingCheckRegion {
+class PollingOverrider {
  public:
-  explicit PollingCheckRegion(bool allow_blocking) {}
+  explicit PollingOverrider(bool allow_blocking) {}
 };
 #endif
 
-class Verifier : public PollingCheckRegion {
+class Verifier {
  public:
-  explicit Verifier(bool spin) : PollingCheckRegion(!spin), spin_(spin) {}
+  explicit Verifier(bool spin) : spin_(spin) {}
   Verifier& Expect(int i, bool expect_ok) {
     expectations_[tag(i)] = expect_ok;
     return *this;
@@ -183,6 +188,8 @@
   AsyncEnd2endTest() {}
 
   void SetUp() GRPC_OVERRIDE {
+    poll_overrider_.reset(new PollingOverrider(!GetParam()));
+
     int port = grpc_pick_unused_port_or_die();
     server_address_ << "localhost:" << port;
 
@@ -193,6 +200,8 @@
     builder.RegisterService(&service_);
     cq_ = builder.AddCompletionQueue();
     server_ = builder.BuildAndStart();
+
+    gpr_tls_set(&g_is_async_end2end_test, 1);
   }
 
   void TearDown() GRPC_OVERRIDE {
@@ -202,6 +211,8 @@
     cq_->Shutdown();
     while (cq_->Next(&ignored_tag, &ignored_ok))
       ;
+    poll_overrider_.reset();
+    gpr_tls_set(&g_is_async_end2end_test, 0);
   }
 
   void ResetStub() {
@@ -249,6 +260,8 @@
   std::unique_ptr<Server> server_;
   grpc::testing::EchoTestService::AsyncService service_;
   std::ostringstream server_address_;
+
+  std::unique_ptr<PollingOverrider> poll_overrider_;
 };
 
 TEST_P(AsyncEnd2endTest, SimpleRpc) {
@@ -1087,7 +1100,7 @@
     Verifier(GetParam()).Expect(7, true).Verify(cq_.get());
 
     // This is expected to fail in all cases i.e for all values of
-    // server_try_cancel. This is becasue at this point, either there are no
+    // server_try_cancel. This is because at this point, either there are no
     // more msgs from the client (because client called WritesDone) or the RPC
     // is cancelled on the server
     srv_stream.Read(&recv_request, tag(8));
@@ -1164,6 +1177,9 @@
 
 int main(int argc, char** argv) {
   grpc_test_init(argc, argv);
+  gpr_tls_init(&g_is_async_end2end_test);
   ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
+  int ret = RUN_ALL_TESTS();
+  gpr_tls_destroy(&g_is_async_end2end_test);
+  return ret;
 }
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index c852384..ce8e4d2 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -51,11 +51,11 @@
 #include "src/core/security/credentials.h"
 #include "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h"
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
-#include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
 #include "test/cpp/end2end/test_service_impl.h"
 #include "test/cpp/util/string_ref_helper.h"
+#include "test/cpp/util/test_credentials_provider.h"
 
 using grpc::testing::EchoRequest;
 using grpc::testing::EchoResponse;
@@ -191,12 +191,14 @@
 
 class TestScenario {
  public:
-  TestScenario(bool proxy, bool tls) : use_proxy(proxy), use_tls(tls) {}
+  TestScenario(bool proxy, const grpc::string& creds_type)
+      : use_proxy(proxy), credentials_type(creds_type) {}
   void Log() const {
-    gpr_log(GPR_INFO, "Scenario: proxy %d, tls %d", use_proxy, use_tls);
+    gpr_log(GPR_INFO, "Scenario: proxy %d, credentials %s", use_proxy,
+            credentials_type.c_str());
   }
   bool use_proxy;
-  bool use_tls;
+  const grpc::string credentials_type;
 };
 
 class End2endTest : public ::testing::TestWithParam<TestScenario> {
@@ -220,14 +222,8 @@
     server_address_ << "127.0.0.1:" << port;
     // Setup server
     ServerBuilder builder;
-    auto server_creds = InsecureServerCredentials();
-    if (GetParam().use_tls) {
-      SslServerCredentialsOptions::PemKeyCertPair pkcp = {test_server1_key,
-                                                          test_server1_cert};
-      SslServerCredentialsOptions ssl_opts;
-      ssl_opts.pem_root_certs = "";
-      ssl_opts.pem_key_cert_pairs.push_back(pkcp);
-      server_creds = SslServerCredentials(ssl_opts);
+    auto server_creds = GetServerCredentials(GetParam().credentials_type);
+    if (GetParam().credentials_type != kInsecureCredentialsType) {
       server_creds->SetAuthMetadataProcessor(processor);
     }
     builder.AddListeningPort(server_address_.str(), server_creds);
@@ -246,12 +242,8 @@
     }
     EXPECT_TRUE(is_server_started_);
     ChannelArguments args;
-    auto channel_creds = InsecureChannelCredentials();
-    if (GetParam().use_tls) {
-      SslCredentialsOptions ssl_opts = {test_root_cert, "", ""};
-      args.SetSslTargetNameOverride("foo.test.google.fr");
-      channel_creds = SslCredentials(ssl_opts);
-    }
+    auto channel_creds =
+        GetChannelCredentials(GetParam().credentials_type, &args);
     if (!user_agent_prefix_.empty()) {
       args.SetUserAgentPrefix(user_agent_prefix_);
     }
@@ -941,7 +933,7 @@
 
 // Takes 10s.
 TEST_P(End2endTest, ChannelStateTimeout) {
-  if (GetParam().use_tls) {
+  if (GetParam().credentials_type != kInsecureCredentialsType) {
     return;
   }
   int port = grpc_pick_unused_port_or_die();
@@ -1150,7 +1142,7 @@
  protected:
   SecureEnd2endTest() {
     GPR_ASSERT(!GetParam().use_proxy);
-    GPR_ASSERT(GetParam().use_tls);
+    GPR_ASSERT(GetParam().credentials_type != kInsecureCredentialsType);
   }
 };
 
@@ -1373,21 +1365,42 @@
   EXPECT_EQ("*.test.youtube.com", ToString(auth_ctx->GetPeerIdentity()[2]));
 }
 
+std::vector<TestScenario> CreateTestScenarios(bool use_proxy,
+                                              bool test_insecure,
+                                              bool test_secure) {
+  std::vector<TestScenario> scenarios;
+  std::vector<grpc::string> credentials_types;
+  if (test_secure) {
+    credentials_types = GetSecureCredentialsTypeList();
+  }
+  if (test_insecure) {
+    credentials_types.push_back(kInsecureCredentialsType);
+  }
+  for (auto it = credentials_types.begin(); it != credentials_types.end();
+       ++it) {
+    scenarios.push_back(TestScenario(false, *it));
+    if (use_proxy) {
+      scenarios.push_back(TestScenario(true, *it));
+    }
+  }
+  return scenarios;
+}
+
 INSTANTIATE_TEST_CASE_P(End2end, End2endTest,
-                        ::testing::Values(TestScenario(false, false),
-                                          TestScenario(false, true)));
+                        ::testing::ValuesIn(CreateTestScenarios(false, true,
+                                                                true)));
 
 INSTANTIATE_TEST_CASE_P(End2endServerTryCancel, End2endServerTryCancelTest,
-                        ::testing::Values(TestScenario(false, false)));
+                        ::testing::ValuesIn(CreateTestScenarios(false, true,
+                                                                false)));
 
 INSTANTIATE_TEST_CASE_P(ProxyEnd2end, ProxyEnd2endTest,
-                        ::testing::Values(TestScenario(false, false),
-                                          TestScenario(false, true),
-                                          TestScenario(true, false),
-                                          TestScenario(true, true)));
+                        ::testing::ValuesIn(CreateTestScenarios(true, true,
+                                                                true)));
 
 INSTANTIATE_TEST_CASE_P(SecureEnd2end, SecureEnd2endTest,
-                        ::testing::Values(TestScenario(false, true)));
+                        ::testing::ValuesIn(CreateTestScenarios(false, false,
+                                                                true)));
 
 }  // namespace
 }  // namespace testing
diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc
new file mode 100644
index 0000000..1086e14
--- /dev/null
+++ b/test/cpp/util/test_credentials_provider.cc
@@ -0,0 +1,82 @@
+
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/cpp/util/test_credentials_provider.h"
+
+#include "test/core/end2end/data/ssl_test_data.h"
+
+namespace grpc {
+namespace testing {
+
+const char kTlsCredentialsType[] = "TLS_CREDENTIALS";
+
+std::shared_ptr<ChannelCredentials> GetChannelCredentials(
+    const grpc::string& type, ChannelArguments* args) {
+  if (type == kInsecureCredentialsType) {
+    return InsecureChannelCredentials();
+  } else if (type == kTlsCredentialsType) {
+    SslCredentialsOptions ssl_opts = {test_root_cert, "", ""};
+    args->SetSslTargetNameOverride("foo.test.google.fr");
+    return SslCredentials(ssl_opts);
+  } else {
+    gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str());
+  }
+  return nullptr;
+}
+
+std::shared_ptr<ServerCredentials> GetServerCredentials(
+    const grpc::string& type) {
+  if (type == kInsecureCredentialsType) {
+    return InsecureServerCredentials();
+  } else if (type == kTlsCredentialsType) {
+    SslServerCredentialsOptions::PemKeyCertPair pkcp = {test_server1_key,
+                                                        test_server1_cert};
+    SslServerCredentialsOptions ssl_opts;
+    ssl_opts.pem_root_certs = "";
+    ssl_opts.pem_key_cert_pairs.push_back(pkcp);
+    return SslServerCredentials(ssl_opts);
+  } else {
+    gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str());
+  }
+  return nullptr;
+}
+
+std::vector<grpc::string> GetSecureCredentialsTypeList() {
+  std::vector<grpc::string> types;
+  types.push_back(kTlsCredentialsType);
+  return types;
+}
+
+}  // namespace testing
+}  // namespace grpc
diff --git a/test/cpp/util/test_credentials_provider.h b/test/cpp/util/test_credentials_provider.h
new file mode 100644
index 0000000..f725305
--- /dev/null
+++ b/test/cpp/util/test_credentials_provider.h
@@ -0,0 +1,63 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_TEST_CPP_UTIL_TEST_CREDENTIALS_PROVIDER_H
+#define GRPC_TEST_CPP_UTIL_TEST_CREDENTIALS_PROVIDER_H
+
+#include <memory>
+
+#include <grpc++/security/credentials.h>
+#include <grpc++/security/server_credentials.h>
+#include <grpc++/support/channel_arguments.h>
+
+namespace grpc {
+namespace testing {
+
+const char kInsecureCredentialsType[] = "INSECURE_CREDENTIALS";
+
+// Provide channel credentials according to the given type. Alter the channel
+// arguments if needed.
+std::shared_ptr<ChannelCredentials> GetChannelCredentials(
+    const grpc::string& type, ChannelArguments* args);
+
+// Provide server credentials according to the given type.
+std::shared_ptr<ServerCredentials> GetServerCredentials(
+    const grpc::string& type);
+
+// Provide a list of secure credentials type.
+std::vector<grpc::string> GetSecureCredentialsTypeList();
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif  // GRPC_TEST_CPP_UTIL_TEST_CREDENTIALS_PROVIDER_H
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 5b1b674..80edd6d 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -4186,7 +4186,8 @@
       "test/cpp/util/cli_call.h", 
       "test/cpp/util/create_test_channel.h", 
       "test/cpp/util/string_ref_helper.h", 
-      "test/cpp/util/subprocess.h"
+      "test/cpp/util/subprocess.h", 
+      "test/cpp/util/test_credentials_provider.h"
     ], 
     "language": "c++", 
     "name": "grpc++_test_util", 
@@ -4202,7 +4203,9 @@
       "test/cpp/util/string_ref_helper.cc", 
       "test/cpp/util/string_ref_helper.h", 
       "test/cpp/util/subprocess.cc", 
-      "test/cpp/util/subprocess.h"
+      "test/cpp/util/subprocess.h", 
+      "test/cpp/util/test_credentials_provider.cc", 
+      "test/cpp/util/test_credentials_provider.h"
     ]
   }, 
   {
diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj
index 3d35371..33860af 100644
--- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj
+++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj
@@ -153,6 +153,7 @@
     <ClInclude Include="$(SolutionDir)\..\test\cpp\util\create_test_channel.h" />
     <ClInclude Include="$(SolutionDir)\..\test\cpp\util\string_ref_helper.h" />
     <ClInclude Include="$(SolutionDir)\..\test\cpp\util\subprocess.h" />
+    <ClInclude Include="$(SolutionDir)\..\test\cpp\util\test_credentials_provider.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="$(SolutionDir)\..\src\proto\grpc\testing\echo_messages.pb.cc">
@@ -191,6 +192,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\test\cpp\util\subprocess.cc">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\test\cpp\util\test_credentials_provider.cc">
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\grpc++\grpc++.vcxproj">
diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters
index 27ac675..b35ba1f 100644
--- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters
+++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters
@@ -28,6 +28,9 @@
     <ClCompile Include="$(SolutionDir)\..\test\cpp\util\subprocess.cc">
       <Filter>test\cpp\util</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\test\cpp\util\test_credentials_provider.cc">
+      <Filter>test\cpp\util</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(SolutionDir)\..\test\cpp\end2end\test_service_impl.h">
@@ -48,6 +51,9 @@
     <ClInclude Include="$(SolutionDir)\..\test\cpp\util\subprocess.h">
       <Filter>test\cpp\util</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\test\cpp\util\test_credentials_provider.h">
+      <Filter>test\cpp\util</Filter>
+    </ClInclude>
   </ItemGroup>
 
   <ItemGroup>