Reland "iOS: Save perf results under Documents/perf_result.json"
This will require a manual roll to downstream projects, since
the //test:perf_test target was introduced.
This is a reland of 10a8e7a9b5261a7e3ce19900ba3511be3b5911f8
Original change's description:
> iOS: Save perf results under Documents/perf_result.json
>
> TBR=henrika@webrtc.org
>
> Bug: webrtc:7156
> Change-Id: Ib00992cce0007e0b5c9274340df1a892f810b0c5
> Reviewed-on: https://webrtc-review.googlesource.com/29202
> Commit-Queue: Edward Lemur <ehmaldonado@webrtc.org>
> Reviewed-by: Henrik Andreassson <henrika@webrtc.org>
> Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#21244}
R=henrika@webrtc.org, phoglund@webrtc.org
Bug: webrtc:7156
Change-Id: I85fc7bc5fce0894af90017b71b9952b61b523424
Reviewed-on: https://webrtc-review.googlesource.com/37643
Reviewed-by: Patrik Höglund <phoglund@google.com>
Reviewed-by: Henrik Andreassson <henrika@webrtc.org>
Commit-Queue: Edward Lemur <ehmaldonado@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21518}diff --git a/call/BUILD.gn b/call/BUILD.gn
index 640956e..99de0c9 100644
--- a/call/BUILD.gn
+++ b/call/BUILD.gn
@@ -262,6 +262,7 @@
"../test:direct_transport",
"../test:fake_audio_device",
"../test:field_trial",
+ "../test:perf_test",
"../test:test_common",
"../test:test_support",
"../test:video_test_common",
diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn
index 3a4bca1..1d6a70c8 100644
--- a/modules/audio_coding/BUILD.gn
+++ b/modules/audio_coding/BUILD.gn
@@ -1390,6 +1390,7 @@
"../../rtc_base:rtc_base_approved",
"../../system_wrappers",
"../../system_wrappers:field_trial_api",
+ "../../test:perf_test",
"../../test:test_support",
]
@@ -1895,6 +1896,7 @@
deps = [
":isac_fix",
+ "../../test:perf_test",
"../../test:test_support",
]
diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn
index 24658f2..c5724e5 100644
--- a/modules/audio_processing/BUILD.gn
+++ b/modules/audio_processing/BUILD.gn
@@ -719,6 +719,7 @@
"../../rtc_base:protobuf_utils",
"../../rtc_base:rtc_base_approved",
"../../system_wrappers",
+ "../../test:perf_test",
"../../test:test_support",
]
diff --git a/modules/remote_bitrate_estimator/BUILD.gn b/modules/remote_bitrate_estimator/BUILD.gn
index 89e74c4..0ca6b87 100644
--- a/modules/remote_bitrate_estimator/BUILD.gn
+++ b/modules/remote_bitrate_estimator/BUILD.gn
@@ -151,6 +151,7 @@
"../../rtc_base:rtc_numerics",
"../../system_wrappers",
"../../system_wrappers:field_trial_api",
+ "../../test:perf_test",
"../../test:test_support",
"../../voice_engine",
"../bitrate_controller",
diff --git a/rtc_base/unittest_main.cc b/rtc_base/unittest_main.cc
index 80b988d..330bd3f 100644
--- a/rtc_base/unittest_main.cc
+++ b/rtc_base/unittest_main.cc
@@ -109,7 +109,7 @@
rtc::SSLStreamAdapter::enable_time_callback_for_testing();
#if defined(WEBRTC_IOS)
- rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv);
+ rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv, false);
rtc::test::RunTestsFromIOSApp();
#endif
const int res = RUN_ALL_TESTS();
diff --git a/test/BUILD.gn b/test/BUILD.gn
index a10c6af..fe8a89e 100644
--- a/test/BUILD.gn
+++ b/test/BUILD.gn
@@ -118,6 +118,19 @@
]
}
+rtc_source_set("perf_test") {
+ testonly = true
+ sources = [
+ "testsupport/perf_test.cc",
+ "testsupport/perf_test.h",
+ ]
+ deps = [
+ "..:webrtc_common",
+ "../api:array_view",
+ "../rtc_base:rtc_base_approved",
+ ]
+}
+
if (is_ios) {
rtc_source_set("test_support_objc") {
testonly = true
@@ -127,6 +140,7 @@
"ios/test_support.mm",
]
deps = [
+ ":perf_test",
"../sdk:common_objc",
]
}
@@ -140,8 +154,6 @@
"gtest.h",
"testsupport/packet_reader.cc",
"testsupport/packet_reader.h",
- "testsupport/perf_test.cc",
- "testsupport/perf_test.h",
"testsupport/unittest_utils.h",
]
@@ -194,6 +206,7 @@
]
deps = [
":field_trial",
+ ":perf_test",
"../rtc_base:rtc_base_approved",
"../system_wrappers:metrics_default",
"//testing/gmock",
@@ -295,6 +308,7 @@
rtc_test("test_support_unittests") {
deps = [
":fake_audio_device",
+ ":perf_test",
":rtp_test_utils",
"../api:video_frame_api",
"../api:video_frame_api_i420",
diff --git a/test/ios/test_support.h b/test/ios/test_support.h
index 7bc2993..0178fca 100644
--- a/test/ios/test_support.h
+++ b/test/ios/test_support.h
@@ -17,7 +17,8 @@
// This is necessary as iOS doesn't like processes without a gui
// running for longer than a few seconds.
void RunTestsFromIOSApp();
-void InitTestSuite(int (*test_suite)(void), int argc, char* argv[]);
+void InitTestSuite(int (*test_suite)(void), int argc, char* argv[],
+ bool save_chartjson_result);
} // namespace test
} // namespace rtc
diff --git a/test/ios/test_support.mm b/test/ios/test_support.mm
index 603bc8c..5b7f2f9 100644
--- a/test/ios/test_support.mm
+++ b/test/ios/test_support.mm
@@ -11,8 +11,10 @@
#import <UIKit/UIKit.h>
#include "test/ios/test_support.h"
+#include "test/testsupport/perf_test.h"
#import "sdk/objc/Framework/Classes/Common/RTCUIApplicationStatusObserver.h"
+#include "sdk/objc/Framework/Classes/Common/helpers.h"
// Springboard will kill any iOS app that fails to check in after launch within
// a given time. Starting a UIApplication before invoking TestSuite::Run
@@ -31,6 +33,7 @@
static int (*g_test_suite)(void) = NULL;
static int g_argc;
static char **g_argv;
+static bool g_save_chartjson_result;
@interface UIApplication (Testing)
- (void)_terminateWithStatus:(int)status;
@@ -75,6 +78,20 @@
- (void)runTests {
int exitStatus = g_test_suite();
+ if (g_save_chartjson_result) {
+ // Stores data into a json file under the app's document directory.
+ NSString* fileName = @"perf_result.json";
+ NSArray<NSString*>* outputDirectories = NSSearchPathForDirectoriesInDomains(
+ NSDocumentDirectory, NSUserDomainMask, YES);
+ if ([outputDirectories count] != 0) {
+ NSString* outputPath =
+ [outputDirectories[0] stringByAppendingPathComponent:fileName];
+
+ webrtc::test::WritePerfResults(
+ webrtc::ios::StdStringFromNSString(outputPath));
+ }
+ }
+
// If a test app is too fast, it will exit before Instruments has has a
// a chance to initialize and no test results will be seen.
// TODO(crbug.com/137010): Figure out how much time is actually needed, and
@@ -93,10 +110,14 @@
namespace rtc {
namespace test {
-void InitTestSuite(int (*test_suite)(void), int argc, char *argv[]) {
+// Note: This is not thread safe, and must be called from the same thread as
+// runTests above.
+void InitTestSuite(int (*test_suite)(void), int argc, char *argv[],
+ bool save_chartjson_result) {
g_test_suite = test_suite;
g_argc = argc;
g_argv = argv;
+ g_save_chartjson_result = save_chartjson_result;
}
void RunTestsFromIOSApp() {
diff --git a/test/test_main.cc b/test/test_main.cc
index 5023ae6..60e75bc 100644
--- a/test/test_main.cc
+++ b/test/test_main.cc
@@ -8,7 +8,6 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "rtc_base/file.h"
#include "rtc_base/flags.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/metrics_default.h"
@@ -25,6 +24,24 @@
"Intentionally ignored flag intended for iOS simulator.");
DEFINE_string(ApplePersistenceIgnoreState, "",
"Intentionally ignored flag intended for iOS simulator.");
+DEFINE_bool(
+ save_chartjson_result,
+ false,
+ "Store the perf results in Documents/perf_result.json in the format "
+ "described by "
+ "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
+ "data-format.md.");
+
+#else
+
+DEFINE_string(
+ chartjson_result_file,
+ "",
+ "Path where the perf results should be stored in the JSON format described "
+ "by "
+ "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
+ "data-format.md.");
+
#endif
DEFINE_bool(logs, false, "print logs to stderr");
@@ -34,14 +51,6 @@
"E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/"
" will assign the group Enable to field trial WebRTC-FooFeature.");
-DEFINE_string(
- perf_results_json_path,
- "",
- "Path where the perf results should be stored it the JSON format described "
- "by "
- "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
- "data-format.md.");
-
DEFINE_bool(help, false, "Print this message.");
int main(int argc, char* argv[]) {
@@ -65,22 +74,25 @@
webrtc::test::InitFieldTrialsFromString(fieldtrials);
webrtc::metrics::Enable();
+
rtc::LogMessage::SetLogToStderr(FLAG_logs);
+
#if defined(WEBRTC_IOS)
- rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv);
+
+ rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv,
+ FLAG_save_chartjson_result);
rtc::test::RunTestsFromIOSApp();
-#endif
+
+#else
int exit_code = RUN_ALL_TESTS();
- std::string perf_results_json_path = FLAG_perf_results_json_path;
- if (perf_results_json_path != "") {
- std::string json_results = webrtc::test::GetPerfResultsJSON();
- rtc::File json_file = rtc::File::Open(perf_results_json_path);
- json_file.Write(reinterpret_cast<const uint8_t*>(json_results.c_str()),
- json_results.size());
- json_file.Close();
+ std::string chartjson_result_file = FLAG_chartjson_result_file;
+ if (chartjson_result_file != "") {
+ webrtc::test::WritePerfResults(chartjson_result_file);
}
return exit_code;
+
+#endif
}
diff --git a/test/testsupport/perf_test.cc b/test/testsupport/perf_test.cc
index 060cf37..015e140 100644
--- a/test/testsupport/perf_test.cc
+++ b/test/testsupport/perf_test.cc
@@ -12,6 +12,7 @@
#include "rtc_base/criticalsection.h"
#include <stdio.h>
+#include <fstream>
#include <map>
#include <sstream>
#include <vector>
@@ -154,6 +155,13 @@
return GetPerfResultsLogger().ToJSON();
}
+void WritePerfResults(const std::string& output_path) {
+ std::string json_results = GetPerfResultsJSON();
+ std::fstream json_file(output_path, std::fstream::out);
+ json_file << json_results;
+ json_file.close();
+}
+
void PrintResult(const std::string& measurement,
const std::string& modifier,
const std::string& trace,
diff --git a/test/testsupport/perf_test.h b/test/testsupport/perf_test.h
index 1a72c04..f28dd2b 100644
--- a/test/testsupport/perf_test.h
+++ b/test/testsupport/perf_test.h
@@ -61,8 +61,10 @@
const std::string& units,
bool important);
-// Get all perf results to date in a JSON format as described in
+// Write all perf results to date to a JSON file formatted as described in
// https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md
+void WritePerfResults(const std::string& output_path);
+
std::string GetPerfResultsJSON();
// You shouldn't use this function. It's only used to test the functions above.
diff --git a/tools_webrtc/ios/tests/perf_tests.json b/tools_webrtc/ios/tests/perf_tests.json
index f873221..b871173 100644
--- a/tools_webrtc/ios/tests/perf_tests.json
+++ b/tools_webrtc/ios/tests/perf_tests.json
@@ -3,7 +3,7 @@
{
"app": "webrtc_perf_tests",
"test args": [
- "--perf_results_json_path=Documents/perf_result.json"
+ "--save_chartjson_result"
],
"bot_id": "build15-a7",
"pool": "WebRTC"
diff --git a/tools_webrtc/ios/tests/perf_trybot_tests.json b/tools_webrtc/ios/tests/perf_trybot_tests.json
index 3b53e94..69efd54 100644
--- a/tools_webrtc/ios/tests/perf_trybot_tests.json
+++ b/tools_webrtc/ios/tests/perf_trybot_tests.json
@@ -2,6 +2,9 @@
"tests": [
{
"app": "webrtc_perf_tests",
+ "test args": [
+ "--save_chartjson_result"
+ ],
"bot_id": "build16-a7",
"pool": "WebRTC"
}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 3c4f18e..552bfb4 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -118,6 +118,7 @@
"../rtc_base:rtc_base_approved",
"../rtc_base:rtc_base_tests_utils",
"../system_wrappers",
+ "../test:perf_test",
"../test:rtp_test_utils",
"../test:test_common",
"../test:test_renderer",
@@ -326,6 +327,7 @@
"../system_wrappers:metrics_default",
"../test:direct_transport",
"../test:field_trial",
+ "../test:perf_test",
"../test:rtp_test_utils",
"../test:test_common",
"../test:test_support",