Merge pull request #6158 from jtattermusch/performance_testing_polishing
Improvement to benchmarking
diff --git a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
index f954ca5..b457275 100644
--- a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
+++ b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
@@ -129,6 +129,7 @@
public ClientRunnerImpl(List<Channel> channels, ClientType clientType, RpcType rpcType, int outstandingRpcsPerChannel, LoadParams loadParams, PayloadConfig payloadConfig, HistogramParams histogramParams)
{
GrpcPreconditions.CheckArgument(outstandingRpcsPerChannel > 0, "outstandingRpcsPerChannel");
+ GrpcPreconditions.CheckNotNull(histogramParams, "histogramParams");
this.channels = new List<Channel>(channels);
this.clientType = clientType;
this.rpcType = rpcType;
diff --git a/src/node/performance/worker.js b/src/node/performance/worker.js
index 98577bd..7ef9b84 100644
--- a/src/node/performance/worker.js
+++ b/src/node/performance/worker.js
@@ -33,6 +33,7 @@
'use strict';
+var console = require('console');
var worker_service_impl = require('./worker_service_impl');
var grpc = require('../../../');
@@ -48,6 +49,7 @@
var address = '0.0.0.0:' + port;
server.bind(address, server_creds);
server.start();
+ console.log('running QPS worker on %s', address);
return server;
}
diff --git a/src/node/performance/worker_service_impl.js b/src/node/performance/worker_service_impl.js
index 17458e4..4b5cb8f 100644
--- a/src/node/performance/worker_service_impl.js
+++ b/src/node/performance/worker_service_impl.js
@@ -34,6 +34,7 @@
'use strict';
var os = require('os');
+var console = require('console');
var BenchmarkClient = require('./benchmark_client');
var BenchmarkServer = require('./benchmark_server');
@@ -49,6 +50,7 @@
switch (request.argtype) {
case 'setup':
var setup = request.setup;
+ console.log('ClientConfig %j', setup);
client = new BenchmarkClient(setup.server_targets,
setup.client_channels,
setup.histogram_params,
@@ -118,6 +120,7 @@
var stats;
switch (request.argtype) {
case 'setup':
+ console.log('ServerConfig %j', request.setup);
server = new BenchmarkServer('[::]', request.setup.port,
request.setup.security_params);
server.start();
diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto
index 458b19c..5db39d0 100644
--- a/src/proto/grpc/testing/control.proto
+++ b/src/proto/grpc/testing/control.proto
@@ -194,3 +194,44 @@
message Scenarios {
repeated Scenario scenarios = 1;
}
+
+// Basic summary that can be computed from ClientStats and ServerStats
+// once the scenario has finished.
+message ScenarioResultSummary
+{
+ // Total number of operations per second over all clients.
+ double qps = 1;
+ // QPS per one server core.
+ double qps_per_server_core = 2;
+ // server load based on system_time (0.85 => 85%)
+ double server_system_time = 3;
+ // server load based on user_time (0.85 => 85%)
+ double server_user_time = 4;
+ // client load based on system_time (0.85 => 85%)
+ double client_system_time = 5;
+ // client load based on user_time (0.85 => 85%)
+ double client_user_time = 6;
+
+ // X% latency percentiles (in nanoseconds)
+ double latency_50 = 7;
+ double latency_90 = 8;
+ double latency_95 = 9;
+ double latency_99 = 10;
+ double latency_999 = 11;
+}
+
+// Results of a single benchmark scenario.
+message ScenarioResult {
+ // Inputs used to run the scenario.
+ Scenario scenario = 1;
+ // Histograms from all clients merged into one histogram.
+ HistogramData latencies = 2;
+ // Client stats for each client
+ repeated ClientStats client_stats = 3;
+ // Server stats for each server
+ repeated ServerStats server_stats = 4;
+ // Number of cores available to each server
+ repeated int32 server_cores = 5;
+ // An after-the-fact computed summary
+ ScenarioResultSummary summary = 6;
+}
diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index c87ad64..9c0649c 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -343,8 +343,8 @@
// Finish a run
std::unique_ptr<ScenarioResult> result(new ScenarioResult);
- result->client_config = result_client_config;
- result->server_config = result_server_config;
+ Histogram merged_latencies;
+
gpr_log(GPR_INFO, "Finishing clients");
for (auto client = &clients[0]; client != &clients[num_clients]; client++) {
GPR_ASSERT(client->stream->Write(client_mark));
@@ -353,9 +353,8 @@
for (auto client = &clients[0]; client != &clients[num_clients]; client++) {
GPR_ASSERT(client->stream->Read(&client_status));
const auto& stats = client_status.stats();
- result->latencies.MergeProto(stats.latencies());
- result->client_resources.emplace_back(
- stats.time_elapsed(), stats.time_user(), stats.time_system(), -1);
+ merged_latencies.MergeProto(stats.latencies());
+ result->add_client_stats()->CopyFrom(stats);
GPR_ASSERT(!client->stream->Read(&client_status));
}
for (auto client = &clients[0]; client != &clients[num_clients]; client++) {
@@ -363,6 +362,8 @@
}
delete[] clients;
+ merged_latencies.FillProto(result->mutable_latencies());
+
gpr_log(GPR_INFO, "Finishing servers");
for (auto server = &servers[0]; server != &servers[num_servers]; server++) {
GPR_ASSERT(server->stream->Write(server_mark));
@@ -370,10 +371,8 @@
}
for (auto server = &servers[0]; server != &servers[num_servers]; server++) {
GPR_ASSERT(server->stream->Read(&server_status));
- const auto& stats = server_status.stats();
- result->server_resources.emplace_back(
- stats.time_elapsed(), stats.time_user(), stats.time_system(),
- server_status.cores());
+ result->add_server_stats()->CopyFrom(server_status.stats());
+ result->add_server_cores(server_status.cores());
GPR_ASSERT(!server->stream->Read(&server_status));
}
for (auto server = &servers[0]; server != &servers[num_servers]; server++) {
diff --git a/test/cpp/qps/driver.h b/test/cpp/qps/driver.h
index 21e5152..3a5cf13 100644
--- a/test/cpp/qps/driver.h
+++ b/test/cpp/qps/driver.h
@@ -41,29 +41,6 @@
namespace grpc {
namespace testing {
-class ResourceUsage {
- public:
- ResourceUsage(double w, double u, double s, int c)
- : wall_time_(w), user_time_(u), system_time_(s), cores_(c) {}
- double wall_time() const { return wall_time_; }
- double user_time() const { return user_time_; }
- double system_time() const { return system_time_; }
- int cores() const { return cores_; }
-
- private:
- double wall_time_;
- double user_time_;
- double system_time_;
- int cores_;
-};
-
-struct ScenarioResult {
- Histogram latencies;
- std::vector<ResourceUsage> client_resources;
- std::vector<ResourceUsage> server_resources;
- ClientConfig client_config;
- ServerConfig server_config;
-};
std::unique_ptr<ScenarioResult> RunScenario(
const grpc::testing::ClientConfig& client_config, size_t num_clients,
diff --git a/test/cpp/qps/qps_driver.cc b/test/cpp/qps/qps_driver.cc
index e412c69..608181f 100644
--- a/test/cpp/qps/qps_driver.cc
+++ b/test/cpp/qps/qps_driver.cc
@@ -85,7 +85,6 @@
using grpc::testing::ClientType;
using grpc::testing::ServerType;
using grpc::testing::RpcType;
-using grpc::testing::ResourceUsage;
using grpc::testing::SecurityParams;
namespace grpc {
diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc
index 8943a43..e9266a5 100644
--- a/test/cpp/qps/qps_json_driver.cc
+++ b/test/cpp/qps/qps_json_driver.cc
@@ -102,12 +102,16 @@
for (int i = 0; i < scenarios.scenarios_size(); i++) {
const Scenario &scenario = scenarios.scenarios(i);
std::cerr << "RUNNING SCENARIO: " << scenario.name() << "\n";
- const auto result =
+ auto result =
RunScenario(scenario.client_config(), scenario.num_clients(),
scenario.server_config(), scenario.num_servers(),
scenario.warmup_seconds(), scenario.benchmark_seconds(),
scenario.spawn_local_worker_count());
+ // Amend the result with scenario config. Eventually we should adjust
+ // RunScenario contract so we don't need to touch the result here.
+ result->mutable_scenario()->CopyFrom(scenario);
+
GetReporter()->ReportQPS(*result);
GetReporter()->ReportQPSPerCore(*result);
GetReporter()->ReportLatency(*result);
diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc
index b230eb4..07ab0a8 100644
--- a/test/cpp/qps/report.cc
+++ b/test/cpp/qps/report.cc
@@ -33,6 +33,11 @@
#include "test/cpp/qps/report.h"
+#include <fstream>
+
+#include <google/protobuf/util/json_util.h>
+#include <google/protobuf/util/type_resolver_util.h>
+
#include <grpc/support/log.h>
#include "test/cpp/qps/driver.h"
#include "test/cpp/qps/stats.h"
@@ -40,10 +45,13 @@
namespace grpc {
namespace testing {
-static double WallTime(ResourceUsage u) { return u.wall_time(); }
-static double UserTime(ResourceUsage u) { return u.user_time(); }
-static double SystemTime(ResourceUsage u) { return u.system_time(); }
-static int Cores(ResourceUsage u) { return u.cores(); }
+static double WallTime(ClientStats s) { return s.time_elapsed(); }
+static double SystemTime(ClientStats s) { return s.time_system(); }
+static double UserTime(ClientStats s) { return s.time_user(); }
+static double ServerWallTime(ServerStats s) { return s.time_elapsed(); }
+static double ServerSystemTime(ServerStats s) { return s.time_system(); }
+static double ServerUserTime(ServerStats s) { return s.time_user(); }
+static int Cores(int n) { return n; }
void CompositeReporter::add(std::unique_ptr<Reporter> reporter) {
reporters_.emplace_back(std::move(reporter));
@@ -74,102 +82,74 @@
}
void GprLogReporter::ReportQPS(const ScenarioResult& result) {
- gpr_log(
- GPR_INFO, "QPS: %.1f",
- result.latencies.Count() / average(result.client_resources, WallTime));
+ Histogram histogram;
+ histogram.MergeProto(result.latencies());
+ gpr_log(GPR_INFO, "QPS: %.1f",
+ histogram.Count() / average(result.client_stats(), WallTime));
}
void GprLogReporter::ReportQPSPerCore(const ScenarioResult& result) {
- auto qps =
- result.latencies.Count() / average(result.client_resources, WallTime);
+ Histogram histogram;
+ histogram.MergeProto(result.latencies());
+ auto qps = histogram.Count() / average(result.client_stats(), WallTime);
gpr_log(GPR_INFO, "QPS: %.1f (%.1f/server core)", qps,
- qps / sum(result.server_resources, Cores));
+ qps / sum(result.server_cores(), Cores));
}
void GprLogReporter::ReportLatency(const ScenarioResult& result) {
+ Histogram histogram;
+ histogram.MergeProto(result.latencies());
gpr_log(GPR_INFO,
"Latencies (50/90/95/99/99.9%%-ile): %.1f/%.1f/%.1f/%.1f/%.1f us",
- result.latencies.Percentile(50) / 1000,
- result.latencies.Percentile(90) / 1000,
- result.latencies.Percentile(95) / 1000,
- result.latencies.Percentile(99) / 1000,
- result.latencies.Percentile(99.9) / 1000);
+ histogram.Percentile(50) / 1000, histogram.Percentile(90) / 1000,
+ histogram.Percentile(95) / 1000, histogram.Percentile(99) / 1000,
+ histogram.Percentile(99.9) / 1000);
}
void GprLogReporter::ReportTimes(const ScenarioResult& result) {
gpr_log(GPR_INFO, "Server system time: %.2f%%",
- 100.0 * sum(result.server_resources, SystemTime) /
- sum(result.server_resources, WallTime));
+ 100.0 * sum(result.server_stats(), ServerSystemTime) /
+ sum(result.server_stats(), ServerWallTime));
gpr_log(GPR_INFO, "Server user time: %.2f%%",
- 100.0 * sum(result.server_resources, UserTime) /
- sum(result.server_resources, WallTime));
+ 100.0 * sum(result.server_stats(), ServerUserTime) /
+ sum(result.server_stats(), ServerWallTime));
gpr_log(GPR_INFO, "Client system time: %.2f%%",
- 100.0 * sum(result.client_resources, SystemTime) /
- sum(result.client_resources, WallTime));
+ 100.0 * sum(result.client_stats(), SystemTime) /
+ sum(result.client_stats(), WallTime));
gpr_log(GPR_INFO, "Client user time: %.2f%%",
- 100.0 * sum(result.client_resources, UserTime) /
- sum(result.client_resources, WallTime));
+ 100.0 * sum(result.client_stats(), UserTime) /
+ sum(result.client_stats(), WallTime));
}
-void PerfDbReporter::ReportQPS(const ScenarioResult& result) {
- auto qps =
- result.latencies.Count() / average(result.client_resources, WallTime);
+void JsonReporter::ReportQPS(const ScenarioResult& result) {
+ std::unique_ptr<google::protobuf::util::TypeResolver> type_resolver(
+ google::protobuf::util::NewTypeResolverForDescriptorPool(
+ "type.googleapis.com",
+ google::protobuf::DescriptorPool::generated_pool()));
+ grpc::string binary;
+ grpc::string json_string;
+ result.SerializeToString(&binary);
+ auto status = BinaryToJsonString(
+ type_resolver.get(), "type.googleapis.com/grpc.testing.ScenarioResult",
+ binary, &json_string);
+ GPR_ASSERT(status.ok());
- perf_db_client_.setQps(qps);
- perf_db_client_.setConfigs(result.client_config, result.server_config);
+ std::ofstream output_file(report_file_);
+ output_file << json_string;
+ output_file.close();
}
-void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) {
- auto qps =
- result.latencies.Count() / average(result.client_resources, WallTime);
-
- auto qps_per_core = qps / sum(result.server_resources, Cores);
-
- perf_db_client_.setQps(qps);
- perf_db_client_.setQpsPerCore(qps_per_core);
- perf_db_client_.setConfigs(result.client_config, result.server_config);
+void JsonReporter::ReportQPSPerCore(const ScenarioResult& result) {
+ // NOP - all reporting is handled by ReportQPS.
}
-void PerfDbReporter::ReportLatency(const ScenarioResult& result) {
- perf_db_client_.setLatencies(result.latencies.Percentile(50) / 1000,
- result.latencies.Percentile(90) / 1000,
- result.latencies.Percentile(95) / 1000,
- result.latencies.Percentile(99) / 1000,
- result.latencies.Percentile(99.9) / 1000);
- perf_db_client_.setConfigs(result.client_config, result.server_config);
+void JsonReporter::ReportLatency(const ScenarioResult& result) {
+ // NOP - all reporting is handled by ReportQPS.
}
-void PerfDbReporter::ReportTimes(const ScenarioResult& result) {
- const double server_system_time = 100.0 *
- sum(result.server_resources, SystemTime) /
- sum(result.server_resources, WallTime);
- const double server_user_time = 100.0 *
- sum(result.server_resources, UserTime) /
- sum(result.server_resources, WallTime);
- const double client_system_time = 100.0 *
- sum(result.client_resources, SystemTime) /
- sum(result.client_resources, WallTime);
- const double client_user_time = 100.0 *
- sum(result.client_resources, UserTime) /
- sum(result.client_resources, WallTime);
-
- perf_db_client_.setTimes(server_system_time, server_user_time,
- client_system_time, client_user_time);
- perf_db_client_.setConfigs(result.client_config, result.server_config);
-}
-
-void PerfDbReporter::SendData() {
- // send data to performance database
- bool data_state =
- perf_db_client_.sendData(hashed_id_, test_name_, sys_info_, tag_);
-
- // check state of data sending
- if (data_state) {
- gpr_log(GPR_INFO, "Data sent to performance database successfully");
- } else {
- gpr_log(GPR_INFO, "Data could not be sent to performance database");
- }
+void JsonReporter::ReportTimes(const ScenarioResult& result) {
+ // NOP - all reporting is handled by ReportQPS.
}
} // namespace testing
diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h
index 5caf3fe..8f04d84 100644
--- a/test/cpp/qps/report.h
+++ b/test/cpp/qps/report.h
@@ -104,33 +104,19 @@
void ReportTimes(const ScenarioResult& result) GRPC_OVERRIDE;
};
-/** Reporter for performance database tool */
-class PerfDbReporter : public Reporter {
+/** Dumps the report to a JSON file. */
+class JsonReporter : public Reporter {
public:
- PerfDbReporter(const string& name, const string& hashed_id,
- const string& test_name, const string& sys_info,
- const string& server_address, const string& tag)
- : Reporter(name),
- hashed_id_(hashed_id),
- test_name_(test_name),
- sys_info_(sys_info),
- tag_(tag) {
- perf_db_client_.init(grpc::CreateChannel(
- server_address, grpc::InsecureChannelCredentials()));
- }
- ~PerfDbReporter() GRPC_OVERRIDE { SendData(); };
+ JsonReporter(const string& name, const string& report_file)
+ : Reporter(name), report_file_(report_file) {}
private:
- PerfDbClient perf_db_client_;
- std::string hashed_id_;
- std::string test_name_;
- std::string sys_info_;
- std::string tag_;
void ReportQPS(const ScenarioResult& result) GRPC_OVERRIDE;
void ReportQPSPerCore(const ScenarioResult& result) GRPC_OVERRIDE;
void ReportLatency(const ScenarioResult& result) GRPC_OVERRIDE;
void ReportTimes(const ScenarioResult& result) GRPC_OVERRIDE;
- void SendData();
+
+ const string report_file_;
};
} // namespace testing
diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc
index 746d3d7..6fc8640 100644
--- a/test/cpp/util/benchmark_config.cc
+++ b/test/cpp/util/benchmark_config.cc
@@ -37,8 +37,8 @@
DEFINE_bool(enable_log_reporter, true,
"Enable reporting of benchmark results through GprLog");
-DEFINE_bool(report_metrics_db, false,
- "True if metrics to be reported to performance database");
+DEFINE_string(scenario_result_file, "",
+ "Write JSON benchmark report to the file specified.");
DEFINE_string(hashed_id, "", "Hash of the user id");
@@ -71,10 +71,9 @@
composite_reporter->add(
std::unique_ptr<Reporter>(new GprLogReporter("LogReporter")));
}
- if (FLAGS_report_metrics_db) {
+ if (FLAGS_scenario_result_file != "") {
composite_reporter->add(std::unique_ptr<Reporter>(
- new PerfDbReporter("PerfDbReporter", FLAGS_hashed_id, FLAGS_test_name,
- FLAGS_sys_info, FLAGS_server_address, FLAGS_tag)));
+ new JsonReporter("JsonReporter", FLAGS_scenario_result_file)));
}
return std::shared_ptr<Reporter>(composite_reporter);
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index 7a82d25..bd10b6f 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -33,6 +33,11 @@
WARMUP_SECONDS=5
BENCHMARK_SECONDS=30
+HISTOGRAM_PARAMS = {
+ 'resolution': 0.01,
+ 'max_possible': 60e9,
+}
+
EMPTY_GENERIC_PAYLOAD = {
'bytebuf_params': {
'req_size': 0,
@@ -83,7 +88,7 @@
secargs = None
yield {
- 'name': 'generic_async_streaming_ping_pong_%s'
+ 'name': 'cpp_generic_async_streaming_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
@@ -98,6 +103,7 @@
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
@@ -110,7 +116,7 @@
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'generic_async_streaming_qps_unconstrained_%s'
+ 'name': 'cpp_generic_async_streaming_qps_unconstrained_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
@@ -125,6 +131,7 @@
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
@@ -137,7 +144,7 @@
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'generic_async_streaming_qps_one_server_core_%s'
+ 'name': 'cpp_generic_async_streaming_qps_one_server_core_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
@@ -152,6 +159,7 @@
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
@@ -164,7 +172,7 @@
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'protobuf_async_qps_unconstrained_%s'
+ 'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
@@ -178,20 +186,20 @@
'load_params': {
'closed_loop': {}
},
- 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
- 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
'async_server_threads': 1,
- 'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'single_channel_throughput_%s'
+ 'name': 'cpp_single_channel_throughput_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
@@ -206,6 +214,7 @@
'closed_loop': {}
},
'payload_config': BIG_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
@@ -218,7 +227,7 @@
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'protobuf_async_ping_pong_%s'
+ 'name': 'cpp_protobuf_async_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
@@ -233,13 +242,13 @@
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
- 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
'async_server_threads': 1,
- 'payload_config': EMPTY_PROTO_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
@@ -262,8 +271,9 @@
def scenarios(self):
# TODO(jtattermusch): add more scenarios
+ secargs = None
yield {
- 'name': 'csharp_async_generic_streaming_ping_pong',
+ 'name': 'csharp_generic_async_streaming_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
@@ -277,17 +287,97 @@
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 0,
'async_server_threads': 1,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
+ yield {
+ 'name': 'csharp_protobuf_async_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'csharp_protobuf_sync_to_async_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS,
+ 'SERVER_LANGUAGE': 'c++' # recognized by run_performance_tests.py
+ }
def __str__(self):
return 'csharp'
@@ -307,8 +397,9 @@
def scenarios(self):
# TODO(jtattermusch): add more scenarios
+ secargs = None
yield {
- 'name': 'node_sync_unary_ping_pong_protobuf',
+ 'name': 'node_protobuf_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
@@ -317,18 +408,18 @@
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
- 'rpc_type': 'STREAMING',
+ 'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
- 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'server_type': 'ASYNC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 0,
'async_server_threads': 1,
- 'payload_config': EMPTY_PROTO_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 51ed35f..b62a428 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -37,10 +37,12 @@
import multiprocessing
import os
import pipes
+import re
import subprocess
import sys
import tempfile
import time
+import traceback
import uuid
import performance.scenario_config as scenario_config
@@ -82,6 +84,8 @@
else:
host_and_port='localhost:%s' % port
+ # TODO(jtattermusch): with some care, we can calculate the right timeout
+ # of a worker from the sum of warmup + benchmark times for all the scenarios
jobspec = jobset.JobSpec(
cmdline=cmdline,
shortname=shortname,
@@ -94,6 +98,7 @@
# setting QPS_WORKERS env variable here makes sure it works with SSH too.
cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver ' % ','.join(workers)
cmd += '--scenarios_json=%s' % pipes.quote(json.dumps({'scenarios': [scenario_json]}))
+ cmd += ' --scenario_result_file=scenario_result.json'
if remote_host:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && %s"' % (user_at_host, cmd)
@@ -221,15 +226,30 @@
for worker_idx, worker in enumerate(workers)]
-def create_scenarios(languages, workers_by_lang, remote_host=None):
+def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*'):
"""Create jobspecs for scenarios to run."""
scenarios = []
for language in languages:
for scenario_json in language.scenarios():
- scenario = create_scenario_jobspec(scenario_json,
- workers_by_lang[str(language)],
- remote_host=remote_host)
- scenarios.append(scenario)
+ if re.search(args.regex, scenario_json['name']):
+ workers = workers_by_lang[str(language)]
+ # 'SERVER_LANGUAGE' is an indicator for this script to pick
+ # a server in different language. It doesn't belong to the Scenario
+ # schema, so we also need to remove it.
+ custom_server_lang = scenario_json.pop('SERVER_LANGUAGE', None)
+ if custom_server_lang:
+ if not workers_by_lang.get(custom_server_lang, []):
+ print 'Warning: Skipping scenario %s as' % scenario_json['name']
+ print('SERVER_LANGUAGE is set to %s yet the language has '
+ 'not been selected with -l' % custom_server_lang)
+ continue
+ for idx in range(0, scenario_json['num_servers']):
+ # replace first X workers by workers of a different language
+ workers[idx] = workers_by_lang[custom_server_lang][idx]
+ scenario = create_scenario_jobspec(scenario_json,
+ workers,
+ remote_host=remote_host)
+ scenarios.append(scenario)
# the very last scenario requests shutting down the workers.
all_workers = [worker
@@ -268,6 +288,8 @@
nargs='+',
default=[],
help='Worker hosts where to start QPS workers.')
+argp.add_argument('-r', '--regex', default='.*', type=str,
+ help='Regex to select scenarios to run.')
args = argp.parse_args()
@@ -295,6 +317,9 @@
qpsworker_jobs = start_qpsworkers(languages, args.remote_worker_host)
+# TODO(jtattermusch): see https://github.com/grpc/grpc/issues/6174
+time.sleep(5)
+
# get list of worker addresses for each language.
worker_addresses = dict([(str(language), []) for language in languages])
for job in qpsworker_jobs:
@@ -303,7 +328,8 @@
try:
scenarios = create_scenarios(languages,
workers_by_lang=worker_addresses,
- remote_host=args.remote_driver_host)
+ remote_host=args.remote_driver_host,
+ regex=args.regex)
if not scenarios:
raise Exception('No scenarios to run')
@@ -318,5 +344,7 @@
jobset.message('FAILED', 'Some of the scenarios failed.',
do_newline=True)
sys.exit(1)
+except:
+ traceback.print_exc()
finally:
finish_qps_workers(qpsworker_jobs)
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index f8c6586..ad03f16 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -22035,7 +22035,7 @@
{
"args": [
"--scenario_json",
- "'{\"name\": \"generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'"
+ "'{\"name\": \"cpp_generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22056,12 +22056,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:generic_async_streaming_ping_pong_secure"
+ "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_secure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'"
+ "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22082,12 +22082,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:generic_async_streaming_qps_unconstrained_secure"
+ "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'"
+ "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22108,12 +22108,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:generic_async_streaming_qps_one_server_core_secure"
+ "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"protobuf_async_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'"
+ "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22134,12 +22134,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:protobuf_async_qps_unconstrained_secure"
+ "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"single_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'"
+ "'{\"name\": \"cpp_single_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22160,12 +22160,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:single_channel_throughput_secure"
+ "shortname": "json_run_localhost:cpp_single_channel_throughput_secure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"protobuf_async_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'"
+ "'{\"name\": \"cpp_protobuf_async_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22186,12 +22186,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:protobuf_async_ping_pong_secure"
+ "shortname": "json_run_localhost:cpp_protobuf_async_ping_pong_secure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'"
+ "'{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22212,12 +22212,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:generic_async_streaming_ping_pong_insecure"
+ "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_insecure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'"
+ "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22238,12 +22238,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:generic_async_streaming_qps_unconstrained_insecure"
+ "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'"
+ "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22264,12 +22264,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:generic_async_streaming_qps_one_server_core_insecure"
+ "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"protobuf_async_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'"
+ "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22290,12 +22290,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:protobuf_async_qps_unconstrained_insecure"
+ "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"single_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'"
+ "'{\"name\": \"cpp_single_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22316,12 +22316,12 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:single_channel_throughput_insecure"
+ "shortname": "json_run_localhost:cpp_single_channel_throughput_insecure"
},
{
"args": [
"--scenario_json",
- "'{\"name\": \"protobuf_async_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'"
+ "'{\"name\": \"cpp_protobuf_async_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@@ -22342,7 +22342,7 @@
"posix",
"windows"
],
- "shortname": "json_run_localhost:protobuf_async_ping_pong_insecure"
+ "shortname": "json_run_localhost:cpp_protobuf_async_ping_pong_insecure"
},
{
"args": [