iOS: Save perf results under Documents/perf_result.json
TBR=henrika@webrtc.org
Bug: webrtc:7156
Change-Id: Ib00992cce0007e0b5c9274340df1a892f810b0c5
Reviewed-on: https://webrtc-review.googlesource.com/29202
Commit-Queue: Edward Lemur <ehmaldonado@webrtc.org>
Reviewed-by: Henrik Andreassson <henrika@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21244}diff --git a/call/BUILD.gn b/call/BUILD.gn
index ac94457..80a74d4 100644
--- a/call/BUILD.gn
+++ b/call/BUILD.gn
@@ -255,6 +255,7 @@
"../test:direct_transport",
"../test:fake_audio_device",
"../test:field_trial",
+ "../test:perf_test",
"../test:test_common",
"../test:test_support",
"../test:video_test_common",
diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn
index 557247f..dbc821f 100644
--- a/modules/audio_coding/BUILD.gn
+++ b/modules/audio_coding/BUILD.gn
@@ -1318,6 +1318,7 @@
"../../rtc_base:rtc_base_approved",
"../../system_wrappers",
"../../system_wrappers:field_trial_api",
+ "../../test:perf_test",
"../../test:test_support",
]
@@ -1806,6 +1807,7 @@
deps = [
":isac_fix",
+ "../../test:perf_test",
"../../test:test_support",
]
diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn
index 9ea74b8..639424b 100644
--- a/modules/audio_processing/BUILD.gn
+++ b/modules/audio_processing/BUILD.gn
@@ -713,6 +713,7 @@
"../../rtc_base:protobuf_utils",
"../../rtc_base:rtc_base_approved",
"../../system_wrappers",
+ "../../test:perf_test",
"../../test:test_support",
]
diff --git a/modules/remote_bitrate_estimator/BUILD.gn b/modules/remote_bitrate_estimator/BUILD.gn
index 2dc4b22..7c32ce1 100644
--- a/modules/remote_bitrate_estimator/BUILD.gn
+++ b/modules/remote_bitrate_estimator/BUILD.gn
@@ -146,6 +146,7 @@
"../../rtc_base:rtc_numerics",
"../../system_wrappers",
"../../system_wrappers:field_trial_api",
+ "../../test:perf_test",
"../../test:test_support",
"../../voice_engine",
"../bitrate_controller",
diff --git a/rtc_base/unittest_main.cc b/rtc_base/unittest_main.cc
index 80b988d..330bd3f 100644
--- a/rtc_base/unittest_main.cc
+++ b/rtc_base/unittest_main.cc
@@ -109,7 +109,7 @@
rtc::SSLStreamAdapter::enable_time_callback_for_testing();
#if defined(WEBRTC_IOS)
- rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv);
+ rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv, false);
rtc::test::RunTestsFromIOSApp();
#endif
const int res = RUN_ALL_TESTS();
diff --git a/test/BUILD.gn b/test/BUILD.gn
index 936d375..a746833 100644
--- a/test/BUILD.gn
+++ b/test/BUILD.gn
@@ -113,6 +113,19 @@
]
}
+rtc_source_set("perf_test") {
+ testonly = true
+ sources = [
+ "testsupport/perf_test.cc",
+ "testsupport/perf_test.h",
+ ]
+ deps = [
+ "..:webrtc_common",
+ "../api:array_view",
+ "../rtc_base:rtc_base_approved",
+ ]
+}
+
if (is_ios) {
rtc_source_set("test_support_objc") {
testonly = true
@@ -122,6 +135,7 @@
"ios/test_support.mm",
]
deps = [
+ ":perf_test",
"../sdk:common_objc",
]
}
@@ -135,8 +149,6 @@
"gtest.h",
"testsupport/packet_reader.cc",
"testsupport/packet_reader.h",
- "testsupport/perf_test.cc",
- "testsupport/perf_test.h",
"testsupport/unittest_utils.h",
]
@@ -188,6 +200,7 @@
]
deps = [
":field_trial",
+ ":perf_test",
"../rtc_base:rtc_base_approved",
"../system_wrappers:metrics_default",
"//testing/gmock",
@@ -287,6 +300,7 @@
rtc_test("test_support_unittests") {
deps = [
":fake_audio_device",
+ ":perf_test",
":rtp_test_utils",
"../api:video_frame_api",
"../api:video_frame_api_i420",
diff --git a/test/ios/test_support.h b/test/ios/test_support.h
index 7bc2993..0178fca 100644
--- a/test/ios/test_support.h
+++ b/test/ios/test_support.h
@@ -17,7 +17,8 @@
// This is necessary as iOS doesn't like processes without a gui
// running for longer than a few seconds.
void RunTestsFromIOSApp();
-void InitTestSuite(int (*test_suite)(void), int argc, char* argv[]);
+void InitTestSuite(int (*test_suite)(void), int argc, char* argv[],
+ bool save_chartjson_result);
} // namespace test
} // namespace rtc
diff --git a/test/ios/test_support.mm b/test/ios/test_support.mm
index 603bc8c..5b7f2f9 100644
--- a/test/ios/test_support.mm
+++ b/test/ios/test_support.mm
@@ -11,8 +11,10 @@
#import <UIKit/UIKit.h>
#include "test/ios/test_support.h"
+#include "test/testsupport/perf_test.h"
#import "sdk/objc/Framework/Classes/Common/RTCUIApplicationStatusObserver.h"
+#include "sdk/objc/Framework/Classes/Common/helpers.h"
// Springboard will kill any iOS app that fails to check in after launch within
// a given time. Starting a UIApplication before invoking TestSuite::Run
@@ -31,6 +33,7 @@
static int (*g_test_suite)(void) = NULL;
static int g_argc;
static char **g_argv;
+static bool g_save_chartjson_result;
@interface UIApplication (Testing)
- (void)_terminateWithStatus:(int)status;
@@ -75,6 +78,20 @@
- (void)runTests {
int exitStatus = g_test_suite();
+ if (g_save_chartjson_result) {
+ // Stores data into a json file under the app's document directory.
+ NSString* fileName = @"perf_result.json";
+ NSArray<NSString*>* outputDirectories = NSSearchPathForDirectoriesInDomains(
+ NSDocumentDirectory, NSUserDomainMask, YES);
+ if ([outputDirectories count] != 0) {
+ NSString* outputPath =
+ [outputDirectories[0] stringByAppendingPathComponent:fileName];
+
+ webrtc::test::WritePerfResults(
+ webrtc::ios::StdStringFromNSString(outputPath));
+ }
+ }
+
// If a test app is too fast, it will exit before Instruments has has a
// a chance to initialize and no test results will be seen.
// TODO(crbug.com/137010): Figure out how much time is actually needed, and
@@ -93,10 +110,14 @@
namespace rtc {
namespace test {
-void InitTestSuite(int (*test_suite)(void), int argc, char *argv[]) {
+// Note: This is not thread safe, and must be called from the same thread as
+// runTests above.
+void InitTestSuite(int (*test_suite)(void), int argc, char *argv[],
+ bool save_chartjson_result) {
g_test_suite = test_suite;
g_argc = argc;
g_argv = argv;
+ g_save_chartjson_result = save_chartjson_result;
}
void RunTestsFromIOSApp() {
diff --git a/test/test_main.cc b/test/test_main.cc
index c6c00c5..4b36271 100644
--- a/test/test_main.cc
+++ b/test/test_main.cc
@@ -8,8 +8,6 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include <fstream>
-
#include "rtc_base/flags.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/metrics_default.h"
@@ -26,6 +24,24 @@
"Intentionally ignored flag intended for iOS simulator.");
DEFINE_string(ApplePersistenceIgnoreState, "",
"Intentionally ignored flag intended for iOS simulator.");
+DEFINE_bool(
+ save_chartjson_result,
+ false,
+ "Store the perf results in Documents/perf_result.json in the format "
+ "described by "
+ "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
+ "data-format.md.");
+
+#else
+
+DEFINE_string(
+ chartjson_result_file,
+ "",
+ "Path where the perf results should be stored in the JSON format described "
+ "by "
+ "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
+ "data-format.md.");
+
#endif
DEFINE_bool(logs, false, "print logs to stderr");
@@ -35,13 +51,6 @@
"E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/"
" will assign the group Enable to field trial WebRTC-FooFeature.");
-DEFINE_string(
- chartjson_result_file,
- "",
- "Path where the perf results should be stored it the JSON format described "
- "by "
- "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
- "data-format.md.");
DEFINE_bool(help, false, "Print this message.");
@@ -66,21 +75,25 @@
webrtc::test::InitFieldTrialsFromString(fieldtrials);
webrtc::metrics::Enable();
+
rtc::LogMessage::SetLogToStderr(FLAG_logs);
+
#if defined(WEBRTC_IOS)
- rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv);
+
+ rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv,
+ FLAG_save_chartjson_result);
rtc::test::RunTestsFromIOSApp();
-#endif
+
+#else
int exit_code = RUN_ALL_TESTS();
std::string chartjson_result_file = FLAG_chartjson_result_file;
if (chartjson_result_file != "") {
- std::string json_results = webrtc::test::GetPerfResultsJSON();
- std::fstream json_file(chartjson_result_file, std::fstream::out);
- json_file << json_results;
- json_file.close();
+ webrtc::test::WritePerfResults(chartjson_result_file);
}
return exit_code;
+
+#endif
}
diff --git a/test/testsupport/perf_test.cc b/test/testsupport/perf_test.cc
index c2a2a50..3719916 100644
--- a/test/testsupport/perf_test.cc
+++ b/test/testsupport/perf_test.cc
@@ -12,6 +12,7 @@
#include "rtc_base/criticalsection.h"
#include <stdio.h>
+#include <fstream>
#include <map>
#include <sstream>
#include <vector>
@@ -154,6 +155,13 @@
return GetPerfResultsLogger().ToJSON();
}
+void WritePerfResults(const std::string& output_path) {
+ std::string json_results = GetPerfResultsJSON();
+ std::fstream json_file(output_path, std::fstream::out);
+ json_file << json_results;
+ json_file.close();
+}
+
void PrintResult(const std::string& measurement,
const std::string& modifier,
const std::string& trace,
diff --git a/test/testsupport/perf_test.h b/test/testsupport/perf_test.h
index 1a72c04..f28dd2b2ac 100644
--- a/test/testsupport/perf_test.h
+++ b/test/testsupport/perf_test.h
@@ -61,8 +61,10 @@
const std::string& units,
bool important);
-// Get all perf results to date in a JSON format as described in
+// Write all perf results to date to a JSON file formatted as described in
// https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md
+void WritePerfResults(const std::string& output_path);
+
std::string GetPerfResultsJSON();
// You shouldn't use this function. It's only used to test the functions above.
diff --git a/tools_webrtc/ios/tests/perf_tests.json b/tools_webrtc/ios/tests/perf_tests.json
index f873221..b871173 100644
--- a/tools_webrtc/ios/tests/perf_tests.json
+++ b/tools_webrtc/ios/tests/perf_tests.json
@@ -3,7 +3,7 @@
{
"app": "webrtc_perf_tests",
"test args": [
- "--perf_results_json_path=Documents/perf_result.json"
+ "--save_chartjson_result"
],
"bot_id": "build15-a7",
"pool": "WebRTC"
diff --git a/tools_webrtc/ios/tests/perf_trybot_tests.json b/tools_webrtc/ios/tests/perf_trybot_tests.json
index 3b53e94..17301ba 100644
--- a/tools_webrtc/ios/tests/perf_trybot_tests.json
+++ b/tools_webrtc/ios/tests/perf_trybot_tests.json
@@ -2,6 +2,10 @@
"tests": [
{
"app": "webrtc_perf_tests",
+ "test args": [
+ "--gtest_filter=AudioEncoderOpusComplexityAdaptationTest.AdaptationOn",
+ "--save_chartjson_result"
+ ],
"bot_id": "build16-a7",
"pool": "WebRTC"
}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index ea31bf5..16e9709 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -298,6 +298,7 @@
"../system_wrappers:metrics_default",
"../test:direct_transport",
"../test:field_trial",
+ "../test:perf_test",
"../test:rtp_test_utils",
"../test:test_common",
"../test:test_support",
