Merge github.com:grpc/grpc into foo
diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh
index dee033a..544e31d 100755
--- a/tools/jenkins/run_performance.sh
+++ b/tools/jenkins/run_performance.sh
@@ -32,7 +32,7 @@
set -ex
# List of benchmarks that provide good signal for analyzing performance changes in pull requests
-BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
+BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
# Enter the gRPC repo root
cd $(dirname $0)/../..
diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py
index 2337f2b..5c8b93d 100755
--- a/tools/profiling/microbenchmarks/bm_diff.py
+++ b/tools/profiling/microbenchmarks/bm_diff.py
@@ -41,6 +41,9 @@
import os
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
import comment_on_pr
+import jobset
+import itertools
+import speedup
def changed_ratio(n, o):
if float(o) <= .0001: o = 0
@@ -101,8 +104,9 @@
help='Which metrics to track')
argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq'])
argp.add_argument('-d', '--diff_base', type=str)
-argp.add_argument('-r', '--repetitions', type=int, default=4)
+argp.add_argument('-r', '--repetitions', type=int, default=30)
argp.add_argument('-p', '--p_threshold', type=float, default=0.01)
+argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count())
args = argp.parse_args()
assert args.diff_base
@@ -117,7 +121,7 @@
def make_cmd(cfg):
return ['make'] + args.benchmarks + [
- 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]
+ 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs]
def build():
subprocess.check_call(['git', 'submodule', 'update'])
@@ -135,27 +139,24 @@
'--benchmark_out_format=json',
'--benchmark_repetitions=%d' % (args.repetitions)
]
- print cmd
- subprocess.check_call(cmd)
+ return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver),
+ verbose_success=True, timeout_seconds=None)
build()
-for bm in args.benchmarks:
- collect1(bm, 'opt', 'new')
- collect1(bm, 'counters', 'new')
+jobset.run(itertools.chain(
+ (collect1(bm, 'opt', 'new') for bm in args.benchmarks),
+ (collect1(bm, 'counters', 'new') for bm in args.benchmarks),
+), maxjobs=args.jobs)
where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
build()
- comparables = []
- for bm in args.benchmarks:
- try:
- collect1(bm, 'opt', 'old')
- collect1(bm, 'counters', 'old')
- comparables.append(bm)
- except subprocess.CalledProcessError, e:
- pass
+ jobset.run(itertools.chain(
+ (collect1(bm, 'opt', 'old') for bm in args.benchmarks),
+ (collect1(bm, 'counters', 'old') for bm in args.benchmarks),
+ ), maxjobs=args.jobs)
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
@@ -180,16 +181,9 @@
new = self.samples[True][f]
old = self.samples[False][f]
if not new or not old: continue
- p = stats.ttest_ind(new, old)[1]
- new_mdn = median(new)
- old_mdn = median(old)
- delta = new_mdn - old_mdn
- ratio = changed_ratio(new_mdn, old_mdn)
- print '%s: new=%r old=%r new_mdn=%f old_mdn=%f delta=%f(%f:%f) ratio=%f(%f:%f) p=%f' % (
- f, new, old, new_mdn, old_mdn, delta, abs(delta), _INTERESTING[f]['abs_diff'], ratio, abs(ratio), _INTERESTING[f]['pct_diff']/100.0, p
- )
- if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']/100.0:
- self.final[f] = delta
+ s = speedup.speedup(new, old)
+ if s:
+ self.final[f] = '%d%%' % s
return self.final.keys()
def skip(self):
@@ -201,7 +195,7 @@
benchmarks = collections.defaultdict(Benchmark)
-for bm in comparables:
+for bm in args.benchmarks:
with open('%s.counters.new.json' % bm) as f:
js_new_ctr = json.loads(f.read())
with open('%s.opt.new.json' % bm) as f:
@@ -226,7 +220,7 @@
for name, bm in benchmarks.items():
print name
really_interesting.update(bm.process())
-fields = [f for f in args.track if f in args.track]
+fields = [f for f in args.track if f in really_interesting]
headers = ['Benchmark'] + fields
rows = []
diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py
new file mode 100644
index 0000000..ff11453
--- /dev/null
+++ b/tools/profiling/microbenchmarks/speedup.py
@@ -0,0 +1,67 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from scipy import stats
+import math
+
+_THRESHOLD = 0.01
+
+def scale(a, mul):
+ return [x*mul for x in a]
+
+def cmp(a, b):
+ return stats.ttest_ind(a, b)
+
+def speedup(new, old):
+ s0, p0 = cmp(new, old)
+ if math.isnan(p0): return 0
+ if s0 == 0: return 0
+ if p0 > _THRESHOLD: return 0
+ if s0 < 0:
+ pct = 1
+ while pct < 101:
+ sp, pp = cmp(new, scale(old, 1 - pct/100.0))
+ if sp > 0: break
+ if pp > _THRESHOLD: break
+ pct += 1
+ return -(pct - 1)
+ else:
+ pct = 1
+ while pct < 101:
+ sp, pp = cmp(new, scale(old, 1 + pct/100.0))
+ if sp < 0: break
+ if pp > _THRESHOLD: break
+ pct += 1
+ return pct - 1
+
+if __name__ == "__main__":
+ new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0]
+ old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0]
+ print speedup(new, old)
+ print speedup(old, new)