Add a function to report perf results in JSON format.
Add support to report perf results in the JSON format specified in [1].
[1] https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md
Bug: webrtc:8566
Change-Id: I25f829a4b012b3e2a3d56d61582a674f780148d0
Reviewed-on: https://webrtc-review.googlesource.com/26031
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Commit-Queue: Edward Lemur <ehmaldonado@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20950}diff --git a/test/BUILD.gn b/test/BUILD.gn
index 9a50e1c..7eb896d 100644
--- a/test/BUILD.gn
+++ b/test/BUILD.gn
@@ -148,6 +148,7 @@
deps = [
"..:webrtc_common",
+ "../api:array_view",
"../common_video",
"../rtc_base:gtest_prod",
"../rtc_base:rtc_base_approved",
diff --git a/test/testsupport/perf_test.cc b/test/testsupport/perf_test.cc
index b7578d1..affd458 100644
--- a/test/testsupport/perf_test.cc
+++ b/test/testsupport/perf_test.cc
@@ -8,14 +8,12 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-// A stripped-down version of Chromium's chrome/test/perf/perf_test.cc.
-// ResultsToString(), PrintResult(size_t value) and AppendResult(size_t value)
-// have been modified. The remainder are identical to the Chromium version.
-
#include "test/testsupport/perf_test.h"
+#include "rtc_base/criticalsection.h"
-#include <sstream>
#include <stdio.h>
+#include <map>
+#include <sstream>
#include <vector>
namespace {
@@ -36,21 +34,131 @@
values.c_str(), units.c_str());
}
+template <typename Container>
+void OutputListToStream(std::ostream* ostream, const Container& values) {
+ const char* sep = "";
+ for (const auto& v : values) {
+ (*ostream) << sep << v;
+ sep = ",";
+ }
+}
+
+class PerfResultsLogger {
+ public:
+ void ClearResults() {
+ rtc::CritScope lock(&crit_);
+ graphs_.clear();
+ }
+ void LogResult(const std::string& graph_name,
+ const std::string& trace_name,
+ const double value,
+ const std::string& units,
+ const bool important) {
+ std::ostringstream value_stream;
+ value_stream << value;
+ PrintResultsImpl(graph_name, trace_name, value_stream.str(), units,
+ important);
+
+ std::ostringstream json_stream;
+ json_stream << '"' << trace_name << R"(":{)";
+ json_stream << R"("type":"scalar",)";
+ json_stream << R"("value":)" << value << ',';
+ json_stream << R"("units":")" << units << R"("})";
+ rtc::CritScope lock(&crit_);
+ graphs_[graph_name].push_back(json_stream.str());
+ }
+ void LogResultMeanAndError(const std::string& graph_name,
+ const std::string& trace_name,
+ const double mean,
+ const double error,
+ const std::string& units,
+ const bool important) {
+ std::ostringstream value_stream;
+ value_stream << '{' << mean << ',' << error << '}';
+ PrintResultsImpl(graph_name, trace_name, value_stream.str(), units,
+ important);
+
+ std::ostringstream json_stream;
+ json_stream << '"' << trace_name << R"(":{)";
+ json_stream << R"("type":"list_of_scalars",)";
+ json_stream << R"("values":[)" << mean << "],";
+ json_stream << R"("std":)" << error << ',';
+ json_stream << R"("units":")" << units << R"("})";
+ rtc::CritScope lock(&crit_);
+ graphs_[graph_name].push_back(json_stream.str());
+ }
+ void LogResultList(const std::string& graph_name,
+ const std::string& trace_name,
+ const rtc::ArrayView<const double> values,
+ const std::string& units,
+ const bool important) {
+ std::ostringstream value_stream;
+ value_stream << '[';
+ OutputListToStream(&value_stream, values);
+ value_stream << ']';
+ PrintResultsImpl(graph_name, trace_name, value_stream.str(), units,
+ important);
+
+ std::ostringstream json_stream;
+ json_stream << '"' << trace_name << R"(":{)";
+ json_stream << R"("type":"list_of_scalars",)";
+ json_stream << R"("values":)" << value_stream.str() << ',';
+ json_stream << R"("units":")" << units << R"("})";
+ rtc::CritScope lock(&crit_);
+ graphs_[graph_name].push_back(json_stream.str());
+ }
+ std::string ToJSON() const;
+
+ private:
+ rtc::CriticalSection crit_;
+ std::map<std::string, std::vector<std::string>> graphs_
+ RTC_GUARDED_BY(&crit_);
+};
+
+std::string PerfResultsLogger::ToJSON() const {
+ std::ostringstream json_stream;
+ json_stream << R"({"format_version":"1.0",)";
+ json_stream << R"("charts":{)";
+ rtc::CritScope lock(&crit_);
+ for (auto graphs_it = graphs_.begin(); graphs_it != graphs_.end();
+ ++graphs_it) {
+ if (graphs_it != graphs_.begin())
+ json_stream << ',';
+ json_stream << '"' << graphs_it->first << "\":";
+ json_stream << '{';
+ OutputListToStream(&json_stream, graphs_it->second);
+ json_stream << '}';
+ }
+ json_stream << "}}";
+ return json_stream.str();
+}
+
+PerfResultsLogger& GetPerfResultsLogger() {
+ static PerfResultsLogger* const logger_ = new PerfResultsLogger();
+ return *logger_;
+}
+
} // namespace
namespace webrtc {
namespace test {
+void ClearPerfResults() {
+ GetPerfResultsLogger().ClearResults();
+}
+
+std::string GetPerfResultsJSON() {
+ return GetPerfResultsLogger().ToJSON();
+}
+
void PrintResult(const std::string& measurement,
const std::string& modifier,
const std::string& trace,
const double value,
const std::string& units,
bool important) {
- std::ostringstream value_stream;
- value_stream << value;
- PrintResultsImpl(measurement + modifier, trace, value_stream.str(), units,
- important);
+ GetPerfResultsLogger().LogResult(measurement + modifier, trace, value, units,
+ important);
}
void PrintResultMeanAndError(const std::string& measurement,
@@ -60,32 +168,18 @@
const double error,
const std::string& units,
bool important) {
- std::ostringstream value_stream;
- value_stream << '{' << mean << ',' << error << '}';
- PrintResultsImpl(measurement + modifier, trace, value_stream.str(), units,
- important);
+ GetPerfResultsLogger().LogResultMeanAndError(measurement + modifier, trace,
+ mean, error, units, important);
}
void PrintResultList(const std::string& measurement,
const std::string& modifier,
const std::string& trace,
- const std::vector<double>& values,
+ const rtc::ArrayView<const double> values,
const std::string& units,
bool important) {
- std::ostringstream value_stream;
- value_stream << '[';
- if (!values.empty()) {
- auto it = values.begin();
- while (true) {
- value_stream << *it;
- if (++it == values.end())
- break;
- value_stream << ',';
- }
- }
- value_stream << ']';
- PrintResultsImpl(measurement + modifier, trace, value_stream.str(), units,
- important);
+ GetPerfResultsLogger().LogResultList(measurement + modifier, trace, values,
+ units, important);
}
} // namespace test
diff --git a/test/testsupport/perf_test.h b/test/testsupport/perf_test.h
index 31b7b13..1a72c04 100644
--- a/test/testsupport/perf_test.h
+++ b/test/testsupport/perf_test.h
@@ -8,16 +8,13 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-// A stripped-down version of Chromium's chrome/test/perf/perf_test.h.
-// Several functions have been removed; the prototypes of the remainder have
-// not been changed.
-
#ifndef TEST_TESTSUPPORT_PERF_TEST_H_
#define TEST_TESTSUPPORT_PERF_TEST_H_
+#include "api/array_view.h"
+
#include <sstream>
#include <string>
-#include <vector>
namespace webrtc {
namespace test {
@@ -53,7 +50,6 @@
const std::string& units,
bool important);
-
// Like PrintResult(), but prints an entire list of results. The |values|
// will generally be a list of comma-separated numbers. A typical
// post-processing step might produce plots of their mean and standard
@@ -61,10 +57,17 @@
void PrintResultList(const std::string& measurement,
const std::string& modifier,
const std::string& trace,
- const std::vector<double>& values,
+ rtc::ArrayView<const double> values,
const std::string& units,
bool important);
+// Get all perf results to date in a JSON format as described in
+// https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md
+std::string GetPerfResultsJSON();
+
+// You shouldn't use this function. It's only used to test the functions above.
+void ClearPerfResults();
+
} // namespace test
} // namespace webrtc
diff --git a/test/testsupport/perf_test_unittest.cc b/test/testsupport/perf_test_unittest.cc
index 5c0f543..f80bcf8 100644
--- a/test/testsupport/perf_test_unittest.cc
+++ b/test/testsupport/perf_test_unittest.cc
@@ -14,30 +14,97 @@
#include "test/gtest.h"
+namespace {
+
+const char* kJsonExpected = R"({
+ "format_version":"1.0",
+ "charts":{
+ "foobar":{
+ "baz_v":{
+ "type":"scalar",
+ "value":7,
+ "units":"widgets"
+ },
+ "baz_me":{
+ "type":"list_of_scalars",
+ "values":[1],
+ "std":2,
+ "units":"lemurs"
+ },
+ "baz_vl":{
+ "type":"list_of_scalars",
+ "values":[1,2,3],
+ "units":"units"
+ }
+ },
+ "measurementmodifier":{
+ "trace":{
+ "type":"scalar",
+ "value":42,
+ "units":"units"
+ }
+ }
+ }
+})";
+
+std::string RemoveSpaces(std::string s) {
+ s.erase(std::remove(s.begin(), s.end(), ' '), s.end());
+ s.erase(std::remove(s.begin(), s.end(), '\n'), s.end());
+ return s;
+}
+
+} // namespace
+
namespace webrtc {
namespace test {
+class PerfTest : public ::testing::Test {
+ protected:
+ void TearDown() override {
+ ClearPerfResults();
+ }
+};
+
+
#if defined(WEBRTC_IOS)
-#define MAYBE_AppendResult DISABLED_AppendResult
+#define MAYBE_TestPrintResult DISABLED_TestPrintResult
#else
-#define MAYBE_AppendResult AppendResult
+#define MAYBE_TestPrintResult TestPrintResult
#endif
-TEST(PerfTest, MAYBE_AppendResult) {
+TEST_F(PerfTest, MAYBE_TestPrintResult) {
testing::internal::CaptureStdout();
- std::string expected = "RESULT measurementmodifier: trace= 42 units\n";
+ std::string expected;
+
+ expected += "RESULT measurementmodifier: trace= 42 units\n";
PrintResult("measurement", "modifier", "trace", 42, "units", false);
- expected += "*RESULT foobar: baz= 7 widgets\n";
- PrintResult("foo", "bar", "baz", 7, "widgets", true);
+ expected += "*RESULT foobar: baz_v= 7 widgets\n";
+ PrintResult("foo", "bar", "baz_v", 7, "widgets", true);
- expected += "RESULT foobar: baz= {1,2} lemurs\n";
- PrintResultMeanAndError("foo", "bar", "baz", 1, 2, "lemurs", false);
+ expected += "RESULT foobar: baz_me= {1,2} lemurs\n";
+ PrintResultMeanAndError("foo", "bar", "baz_me", 1, 2, "lemurs", false);
- expected += "RESULT foobar: baz= [1,2,3] units\n";
- PrintResultList("foo", "bar", "baz", {1, 2, 3}, "units", false);
+ const double kListOfScalars[] = {1, 2, 3};
+ expected += "RESULT foobar: baz_vl= [1,2,3] units\n";
+ PrintResultList("foo", "bar", "baz_vl", kListOfScalars, "units", false);
- std::string output = testing::internal::GetCapturedStdout();
- EXPECT_EQ(expected, output);
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST_F(PerfTest, TestGetPerfResultsJSON) {
+ PrintResult("measurement", "modifier", "trace", 42, "units", false);
+ PrintResult("foo", "bar", "baz_v", 7, "widgets", true);
+ PrintResultMeanAndError("foo", "bar", "baz_me", 1, 2, "lemurs", false);
+ const double kListOfScalars[] = {1, 2, 3};
+ PrintResultList("foo", "bar", "baz_vl", kListOfScalars, "units", false);
+
+ EXPECT_EQ(RemoveSpaces(kJsonExpected), GetPerfResultsJSON());
+}
+
+TEST_F(PerfTest, TestClearPerfResults) {
+ PrintResult("measurement", "modifier", "trace", 42, "units", false);
+ ClearPerfResults();
+ EXPECT_EQ(R"({"format_version":"1.0","charts":{}})", GetPerfResultsJSON());
}
} // namespace test
