Merge pull request #9770 from ctiller/this-was-a-good-idea-that-wasnt

Remove internal api canaries: they have never solved anything for us
diff --git a/Makefile b/Makefile
index 37035f5..6fd788b 100644
--- a/Makefile
+++ b/Makefile
@@ -217,7 +217,7 @@
 CXX_counters = $(DEFAULT_CXX)
 LD_counters = $(DEFAULT_CC)
 LDXX_counters = $(DEFAULT_CXX)
-CPPFLAGS_counters = -O2 -DGPR_MU_COUNTERS
+CPPFLAGS_counters = -O2 -DGPR_LOW_LEVEL_COUNTERS
 DEFINES_counters = NDEBUG
 
 
@@ -1857,8 +1857,6 @@
 
 
 flaky_test_c: buildtests_c
-	$(E) "[RUN]     Testing lb_policies_test"
-	$(Q) $(BINDIR)/$(CONFIG)/lb_policies_test || ( echo test lb_policies_test failed ; exit 1 )
 	$(E) "[RUN]     Testing mlog_test"
 	$(Q) $(BINDIR)/$(CONFIG)/mlog_test || ( echo test mlog_test failed ; exit 1 )
 
diff --git a/build.yaml b/build.yaml
index 39447bf..9c9c8f4 100644
--- a/build.yaml
+++ b/build.yaml
@@ -2324,8 +2324,8 @@
   - gpr
 - name: lb_policies_test
   cpu_cost: 0.1
-  flaky: true
   build: test
+  run: false
   language: c
   src:
   - test/core/client_channel/lb_policies_test.c
@@ -3886,7 +3886,7 @@
     CPPFLAGS: -O2 -DGRPC_BASIC_PROFILER -DGRPC_TIMERS_RDTSC
     DEFINES: NDEBUG
   counters:
-    CPPFLAGS: -O2 -DGPR_MU_COUNTERS
+    CPPFLAGS: -O2 -DGPR_LOW_LEVEL_COUNTERS
     DEFINES: NDEBUG
   dbg:
     CPPFLAGS: -O0
diff --git a/include/grpc/impl/codegen/atm_gcc_atomic.h b/include/grpc/impl/codegen/atm_gcc_atomic.h
index 7d4ae98..4bd3b25 100644
--- a/include/grpc/impl/codegen/atm_gcc_atomic.h
+++ b/include/grpc/impl/codegen/atm_gcc_atomic.h
@@ -40,6 +40,20 @@
 
 typedef intptr_t gpr_atm;
 
+#ifdef GPR_LOW_LEVEL_COUNTERS
+extern gpr_atm gpr_counter_atm_cas;
+extern gpr_atm gpr_counter_atm_add;
+#define GPR_ATM_INC_COUNTER(counter) \
+  __atomic_fetch_add(&counter, 1, __ATOMIC_RELAXED)
+#define GPR_ATM_INC_CAS_THEN(blah) \
+  (GPR_ATM_INC_COUNTER(gpr_counter_atm_cas), blah)
+#define GPR_ATM_INC_ADD_THEN(blah) \
+  (GPR_ATM_INC_COUNTER(gpr_counter_atm_add), blah)
+#else
+#define GPR_ATM_INC_CAS_THEN(blah) blah
+#define GPR_ATM_INC_ADD_THEN(blah) blah
+#endif
+
 #define gpr_atm_full_barrier() (__atomic_thread_fence(__ATOMIC_SEQ_CST))
 
 #define gpr_atm_acq_load(p) (__atomic_load_n((p), __ATOMIC_ACQUIRE))
@@ -50,25 +64,28 @@
   (__atomic_store_n((p), (intptr_t)(value), __ATOMIC_RELAXED))
 
 #define gpr_atm_no_barrier_fetch_add(p, delta) \
-  (__atomic_fetch_add((p), (intptr_t)(delta), __ATOMIC_RELAXED))
+  GPR_ATM_INC_ADD_THEN(                        \
+      __atomic_fetch_add((p), (intptr_t)(delta), __ATOMIC_RELAXED))
 #define gpr_atm_full_fetch_add(p, delta) \
-  (__atomic_fetch_add((p), (intptr_t)(delta), __ATOMIC_ACQ_REL))
+  GPR_ATM_INC_ADD_THEN(                  \
+      __atomic_fetch_add((p), (intptr_t)(delta), __ATOMIC_ACQ_REL))
 
 static __inline int gpr_atm_no_barrier_cas(gpr_atm *p, gpr_atm o, gpr_atm n) {
-  return __atomic_compare_exchange_n(p, &o, n, 0, __ATOMIC_RELAXED,
-                                     __ATOMIC_RELAXED);
+  return GPR_ATM_INC_CAS_THEN(__atomic_compare_exchange_n(
+      p, &o, n, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
 }
 
 static __inline int gpr_atm_acq_cas(gpr_atm *p, gpr_atm o, gpr_atm n) {
-  return __atomic_compare_exchange_n(p, &o, n, 0, __ATOMIC_ACQUIRE,
-                                     __ATOMIC_RELAXED);
+  return GPR_ATM_INC_CAS_THEN(__atomic_compare_exchange_n(
+      p, &o, n, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
 }
 
 static __inline int gpr_atm_rel_cas(gpr_atm *p, gpr_atm o, gpr_atm n) {
-  return __atomic_compare_exchange_n(p, &o, n, 0, __ATOMIC_RELEASE,
-                                     __ATOMIC_RELAXED);
+  return GPR_ATM_INC_CAS_THEN(__atomic_compare_exchange_n(
+      p, &o, n, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED));
 }
 
-#define gpr_atm_full_xchg(p, n) __atomic_exchange_n((p), (n), __ATOMIC_ACQ_REL)
+#define gpr_atm_full_xchg(p, n) \
+  GPR_ATM_INC_CAS_THEN(__atomic_exchange_n((p), (n), __ATOMIC_ACQ_REL))
 
 #endif /* GRPC_IMPL_CODEGEN_ATM_GCC_ATOMIC_H */
diff --git a/src/core/lib/support/sync_posix.c b/src/core/lib/support/sync_posix.c
index de0f048..16e7d6e 100644
--- a/src/core/lib/support/sync_posix.c
+++ b/src/core/lib/support/sync_posix.c
@@ -42,8 +42,10 @@
 #include <time.h>
 #include "src/core/lib/profiling/timers.h"
 
-#ifdef GPR_MU_COUNTERS
-gpr_atm grpc_mu_locks = 0;
+#ifdef GPR_LOW_LEVEL_COUNTERS
+gpr_atm gpr_mu_locks = 0;
+gpr_atm gpr_counter_atm_cas = 0;
+gpr_atm gpr_counter_atm_add = 0;
 #endif
 
 void gpr_mu_init(gpr_mu* mu) { GPR_ASSERT(pthread_mutex_init(mu, NULL) == 0); }
@@ -51,8 +53,8 @@
 void gpr_mu_destroy(gpr_mu* mu) { GPR_ASSERT(pthread_mutex_destroy(mu) == 0); }
 
 void gpr_mu_lock(gpr_mu* mu) {
-#ifdef GPR_MU_COUNTERS
-  gpr_atm_no_barrier_fetch_add(&grpc_mu_locks, 1);
+#ifdef GPR_LOW_LEVEL_COUNTERS
+  GPR_ATM_INC_COUNTER(gpr_mu_locks);
 #endif
   GPR_TIMER_BEGIN("gpr_mu_lock", 0);
   GPR_ASSERT(pthread_mutex_lock(mu) == 0);
diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb
index 6934257..d8de0be 100644
--- a/src/ruby/lib/grpc/generic/client_stub.rb
+++ b/src/ruby/lib/grpc/generic/client_stub.rb
@@ -84,7 +84,7 @@
     # channel:
     #
     # - :channel_override
-    # when present, this must be a pre-created GRPC::Channel.  If it's
+    # when present, this must be a pre-created GRPC::Core::Channel.  If it's
     # present the host and arbitrary keyword arg areignored, and the RPC
     # connection uses this channel.
     #
diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc
index 80d6610..03aede3 100644
--- a/test/cpp/microbenchmarks/bm_closure.cc
+++ b/test/cpp/microbenchmarks/bm_closure.cc
@@ -43,13 +43,53 @@
 
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
+#include <sstream>
+
+#ifdef GPR_LOW_LEVEL_COUNTERS
+extern "C" gpr_atm gpr_mu_locks;
+#endif
+
 static class InitializeStuff {
  public:
   InitializeStuff() { grpc_init(); }
   ~InitializeStuff() { grpc_shutdown(); }
 } initialize_stuff;
 
+class TrackCounters {
+ public:
+  TrackCounters(benchmark::State& state) : state_(state) {}
+
+  ~TrackCounters() {
+    std::ostringstream out;
+#ifdef GPR_LOW_LEVEL_COUNTERS
+    out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
+                                       mu_locks_at_start_) /
+                              (double)state_.iterations())
+        << " atm_cas/iter:"
+        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) -
+                     atm_cas_at_start_) /
+            (double)state_.iterations())
+        << " atm_add/iter:"
+        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
+                     atm_add_at_start_) /
+            (double)state_.iterations());
+#endif
+    state_.SetLabel(out.str());
+  }
+
+ private:
+  benchmark::State& state_;
+#ifdef GPR_LOW_LEVEL_COUNTERS
+  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
+  const size_t atm_cas_at_start_ =
+      gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
+  const size_t atm_add_at_start_ =
+      gpr_atm_no_barrier_load(&gpr_counter_atm_add);
+#endif
+};
+
 static void BM_NoOpExecCtx(benchmark::State& state) {
+  TrackCounters track_counters(state);
   while (state.KeepRunning()) {
     grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
     grpc_exec_ctx_finish(&exec_ctx);
@@ -58,6 +98,7 @@
 BENCHMARK(BM_NoOpExecCtx);
 
 static void BM_WellFlushed(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   while (state.KeepRunning()) {
     grpc_exec_ctx_flush(&exec_ctx);
@@ -69,6 +110,7 @@
 static void DoNothing(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {}
 
 static void BM_ClosureInitAgainstExecCtx(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_closure c;
   while (state.KeepRunning()) {
     benchmark::DoNotOptimize(
@@ -78,6 +120,7 @@
 BENCHMARK(BM_ClosureInitAgainstExecCtx);
 
 static void BM_ClosureInitAgainstCombiner(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -91,6 +134,7 @@
 BENCHMARK(BM_ClosureInitAgainstCombiner);
 
 static void BM_ClosureRunOnExecCtx(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -103,6 +147,7 @@
 BENCHMARK(BM_ClosureRunOnExecCtx);
 
 static void BM_ClosureCreateAndRun(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   while (state.KeepRunning()) {
     grpc_closure_run(&exec_ctx, grpc_closure_create(DoNothing, NULL,
@@ -114,6 +159,7 @@
 BENCHMARK(BM_ClosureCreateAndRun);
 
 static void BM_ClosureInitAndRun(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_closure c;
   while (state.KeepRunning()) {
@@ -126,6 +172,7 @@
 BENCHMARK(BM_ClosureInitAndRun);
 
 static void BM_ClosureSchedOnExecCtx(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -138,6 +185,7 @@
 BENCHMARK(BM_ClosureSchedOnExecCtx);
 
 static void BM_ClosureSched2OnExecCtx(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_closure c1;
   grpc_closure c2;
   grpc_closure_init(&c1, DoNothing, NULL, grpc_schedule_on_exec_ctx);
@@ -153,6 +201,7 @@
 BENCHMARK(BM_ClosureSched2OnExecCtx);
 
 static void BM_ClosureSched3OnExecCtx(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_closure c1;
   grpc_closure c2;
   grpc_closure c3;
@@ -171,6 +220,7 @@
 BENCHMARK(BM_ClosureSched3OnExecCtx);
 
 static void BM_AcquireMutex(benchmark::State& state) {
+  TrackCounters track_counters(state);
   // for comparison with the combiner stuff below
   gpr_mu mu;
   gpr_mu_init(&mu);
@@ -185,6 +235,7 @@
 BENCHMARK(BM_AcquireMutex);
 
 static void BM_ClosureSchedOnCombiner(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL,
@@ -200,6 +251,7 @@
 BENCHMARK(BM_ClosureSchedOnCombiner);
 
 static void BM_ClosureSched2OnCombiner(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c1;
   grpc_closure c2;
@@ -219,6 +271,7 @@
 BENCHMARK(BM_ClosureSched2OnCombiner);
 
 static void BM_ClosureSched3OnCombiner(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c1;
   grpc_closure c2;
@@ -242,6 +295,7 @@
 BENCHMARK(BM_ClosureSched3OnCombiner);
 
 static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_combiner* combiner1 = grpc_combiner_create(NULL);
   grpc_combiner* combiner2 = grpc_combiner_create(NULL);
   grpc_closure c1;
@@ -263,6 +317,7 @@
 BENCHMARK(BM_ClosureSched2OnTwoCombiners);
 
 static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_combiner* combiner1 = grpc_combiner_create(NULL);
   grpc_combiner* combiner2 = grpc_combiner_create(NULL);
   grpc_closure c1;
@@ -323,6 +378,7 @@
 };
 
 static void BM_ClosureReschedOnExecCtx(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   Rescheduler r(state, grpc_schedule_on_exec_ctx);
   r.ScheduleFirst(&exec_ctx);
@@ -331,6 +387,7 @@
 BENCHMARK(BM_ClosureReschedOnExecCtx);
 
 static void BM_ClosureReschedOnCombiner(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   Rescheduler r(state, grpc_combiner_scheduler(combiner, false));
@@ -342,6 +399,7 @@
 BENCHMARK(BM_ClosureReschedOnCombiner);
 
 static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) {
+  TrackCounters track_counters(state);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   Rescheduler r(state, grpc_combiner_finally_scheduler(combiner, false));
diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc
index c63de0c..48e131f 100644
--- a/test/cpp/microbenchmarks/bm_fullstack.cc
+++ b/test/cpp/microbenchmarks/bm_fullstack.cc
@@ -99,8 +99,10 @@
   c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX);
 }
 
-#ifdef GPR_MU_COUNTERS
-extern "C" gpr_atm grpc_mu_locks;
+#ifdef GPR_LOW_LEVEL_COUNTERS
+extern "C" gpr_atm gpr_mu_locks;
+extern "C" gpr_atm gpr_counter_atm_cas;
+extern "C" gpr_atm gpr_counter_atm_add;
 #endif
 
 class BaseFixture {
@@ -108,10 +110,18 @@
   void Finish(benchmark::State& s) {
     std::ostringstream out;
     this->AddToLabel(out, s);
-#ifdef GPR_MU_COUNTERS
-    out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&grpc_mu_locks) -
+#ifdef GPR_LOW_LEVEL_COUNTERS
+    out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
                                        mu_locks_at_start_) /
-                              (double)s.iterations());
+                              (double)s.iterations())
+        << " atm_cas/iter:"
+        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) -
+                     atm_cas_at_start_) /
+            (double)s.iterations())
+        << " atm_add/iter:"
+        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
+                     atm_add_at_start_) /
+            (double)s.iterations());
 #endif
     grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();
     out << " allocs/iter:"
@@ -128,8 +138,12 @@
   virtual void AddToLabel(std::ostream& out, benchmark::State& s) = 0;
 
  private:
-#ifdef GPR_MU_COUNTERS
-  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&grpc_mu_locks);
+#ifdef GPR_LOW_LEVEL_COUNTERS
+  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
+  const size_t atm_cas_at_start_ =
+      gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
+  const size_t atm_add_at_start_ =
+      gpr_atm_no_barrier_load(&gpr_counter_atm_add);
 #endif
   grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot();
 };
diff --git a/test/cpp/qps/qps_test_with_poll.cc b/test/cpp/qps/qps_test_with_poll.cc
deleted file mode 100644
index c64e6c9..0000000
--- a/test/cpp/qps/qps_test_with_poll.cc
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <set>
-
-#include <grpc/support/log.h>
-
-#include "test/cpp/qps/driver.h"
-#include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
-
-extern "C" {
-#include "src/core/lib/iomgr/pollset_posix.h"
-}
-
-namespace grpc {
-namespace testing {
-
-static const int WARMUP = 5;
-static const int BENCHMARK = 5;
-
-static void RunQPS() {
-  gpr_log(GPR_INFO, "Running QPS test");
-
-  ClientConfig client_config;
-  client_config.set_client_type(ASYNC_CLIENT);
-  client_config.set_outstanding_rpcs_per_channel(1000);
-  client_config.set_client_channels(8);
-  client_config.set_async_client_threads(8);
-  client_config.set_rpc_type(UNARY);
-  client_config.mutable_load_params()->mutable_closed_loop();
-
-  ServerConfig server_config;
-  server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_async_server_threads(4);
-
-  const auto result =
-      RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
-
-  GetReporter()->ReportQPSPerCore(*result);
-  GetReporter()->ReportLatency(*result);
-}
-
-}  // namespace testing
-}  // namespace grpc
-
-int main(int argc, char** argv) {
-  grpc::testing::InitBenchmark(&argc, &argv, true);
-
-  grpc_platform_become_multipoller = grpc_poll_become_multipoller;
-
-  grpc::testing::RunQPS();
-
-  return 0;
-}
diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py
index a7d8226..8ead4b4 100755
--- a/tools/profiling/microbenchmarks/bm2bq.py
+++ b/tools/profiling/microbenchmarks/bm2bq.py
@@ -66,6 +66,8 @@
   ('cli_stream_stalls_per_iteration', 'float'),
   ('svr_transport_stalls_per_iteration', 'float'),
   ('svr_stream_stalls_per_iteration', 'float'),
+  ('atm_cas_per_iteration', 'float')
+  ('atm_add_per_iteration', 'float')
 ]
 
 if sys.argv[1] == '--schema':
@@ -158,7 +160,7 @@
 for bm in js['benchmarks']:
   context = js['context']
   if 'label' in bm:
-    labels_list = [s.split(':') for s in bm['label'].split(' ')]
+    labels_list = [s.split(':') for s in bm['label'].strip().split(' ')]
     for el in labels_list:
       el[0] = el[0].replace('/iter', '_per_iteration')
     labels = dict(labels_list)
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index d238fbc..0cfdab4 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -1529,28 +1529,6 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": true, 
-    "gtest": false, 
-    "language": "c", 
-    "name": "lb_policies_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "args": [], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
     "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [],