Add improvement directions to PC and Call framework metrics
Bug: webrtc:10138
Change-Id: Ib957950df6e7490a15da0345fcd73e037c1a5b19
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/153892
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Ivo Creusen <ivoc@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29278}
diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
index 07f3d63..b8f1740 100644
--- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
@@ -12,7 +12,6 @@
#include "api/stats_types.h"
#include "rtc_base/logging.h"
-#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@@ -100,18 +99,21 @@
}
void DefaultAudioQualityAnalyzer::Stop() {
+ using ::webrtc::test::ImproveDirection;
rtc::CritScope crit(&lock_);
for (auto& item : streams_stats_) {
- ReportResult("expand_rate", item.first, item.second.expand_rate,
- "unitless");
+ ReportResult("expand_rate", item.first, item.second.expand_rate, "unitless",
+ ImproveDirection::kSmallerIsBetter);
ReportResult("accelerate_rate", item.first, item.second.accelerate_rate,
- "unitless");
+ "unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("preemptive_rate", item.first, item.second.preemptive_rate,
- "unitless");
+ "unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("speech_expand_rate", item.first,
- item.second.speech_expand_rate, "unitless");
+ item.second.speech_expand_rate, "unitless",
+ ImproveDirection::kSmallerIsBetter);
ReportResult("preferred_buffer_size_ms", item.first,
- item.second.preferred_buffer_size_ms, "ms");
+ item.second.preferred_buffer_size_ms, "ms",
+ ImproveDirection::kNone);
}
}
@@ -125,12 +127,13 @@
const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
- const std::string& unit) const {
+ const std::string& unit,
+ webrtc::test::ImproveDirection improve_direction) const {
test::PrintResultMeanAndError(
metric_name, /*modifier=*/"", GetTestCaseName(stream_label),
counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit,
- /*important=*/false);
+ /*important=*/false, improve_direction);
}
} // namespace webrtc_pc_e2e
diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
index ee34ed3..33aaefd 100644
--- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
+++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
@@ -19,6 +19,7 @@
#include "api/test/track_id_stream_label_map.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/numerics/samples_stats_counter.h"
+#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@@ -51,7 +52,8 @@
void ReportResult(const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
- const std::string& unit) const;
+ const std::string& unit,
+ webrtc::test::ImproveDirection improve_direction) const;
std::string test_case_name_;
TrackIdStreamLabelMap* analyzer_helper_;
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
index cd3a95d..2605461 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -17,7 +17,6 @@
#include "api/units/time_delta.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/logging.h"
-#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@@ -633,49 +632,64 @@
const std::string& test_case_name,
const StreamStats& stats,
const FrameCounters& frame_counters) {
- ReportResult("psnr", test_case_name, stats.psnr, "dB");
- ReportResult("ssim", test_case_name, stats.ssim, "unitless");
- ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms");
+ using ::webrtc::test::ImproveDirection;
+
+ ReportResult("psnr", test_case_name, stats.psnr, "dB",
+ ImproveDirection::kBiggerIsBetter);
+ ReportResult("ssim", test_case_name, stats.ssim, "unitless",
+ ImproveDirection::kBiggerIsBetter);
+ ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
ReportResult("total_delay_incl_transport", test_case_name,
- stats.total_delay_incl_transport_ms, "ms");
+ stats.total_delay_incl_transport_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_rendered_frames", test_case_name,
- stats.time_between_rendered_frames_ms, "ms");
+ stats.time_between_rendered_frames_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
test::PrintResult("encode_frame_rate", "", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
- "fps", /*important=*/false);
- ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms");
+ "fps", /*important=*/false,
+ ImproveDirection::kBiggerIsBetter);
+ ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_freezes", test_case_name,
- stats.time_between_freezes_ms, "ms");
- ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms");
+ stats.time_between_freezes_ms, "ms",
+ ImproveDirection::kBiggerIsBetter);
+ ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
ReportResult("pixels_per_frame", test_case_name,
- stats.resolution_of_rendered_frame, "count");
+ stats.resolution_of_rendered_frame, "count",
+ ImproveDirection::kBiggerIsBetter);
test::PrintResult("min_psnr", "", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
- /*important=*/false);
- ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms");
+ /*important=*/false, ImproveDirection::kBiggerIsBetter);
+ ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
ReportResult("receive_to_render_time", test_case_name,
- stats.receive_to_render_time_ms, "ms");
+ stats.receive_to_render_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
test::PrintResult("dropped_frames", "", test_case_name,
frame_counters.dropped, "count",
- /*important=*/false);
+ /*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("frames_in_flight", "", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
"count",
- /*important=*/false);
+ /*important=*/false, ImproveDirection::kSmallerIsBetter);
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
- "count");
+ "count", ImproveDirection::kSmallerIsBetter);
}
void DefaultVideoQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
- const std::string& unit) {
+ const std::string& unit,
+ webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
- /*important=*/false);
+ /*important=*/false, improve_direction);
}
std::string DefaultVideoQualityAnalyzer::GetTestCaseName(
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
index 5da4cda..d99e6ac 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
@@ -28,6 +28,7 @@
#include "rtc_base/numerics/samples_stats_counter.h"
#include "rtc_base/platform_thread.h"
#include "system_wrappers/include/clock.h"
+#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@@ -248,7 +249,9 @@
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
- const std::string& unit);
+ const std::string& unit,
+ webrtc::test::ImproveDirection improve_direction =
+ webrtc::test::ImproveDirection::kNone);
// Returns name of current test case for reporting.
std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now();
diff --git a/test/testsupport/perf_test.cc b/test/testsupport/perf_test.cc
index 4f7f518..07f5fee 100644
--- a/test/testsupport/perf_test.cc
+++ b/test/testsupport/perf_test.cc
@@ -55,11 +55,12 @@
const std::string& trace_name,
const webrtc::SamplesStatsCounter& counter,
const std::string& units,
- const bool important) {
+ const bool important,
+ webrtc::test::ImproveDirection improve_direction) {
LogResultMeanAndError(
graph_name, trace_name, counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), units,
- important);
+ important, improve_direction);
rtc::CritScope lock(&crit_);
plottable_counters_.push_back({graph_name, trace_name, counter, units});
@@ -68,7 +69,8 @@
const std::string& trace_name,
const double value,
const std::string& units,
- const bool important) {
+ const bool important,
+ webrtc::test::ImproveDirection improve_direction) {
RTC_CHECK(std::isfinite(value))
<< "Expected finite value for graph " << graph_name << ", trace name "
<< trace_name << ", units " << units << ", got " << value;
@@ -76,14 +78,15 @@
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << value;
- LogResultsImpl(graph_name, trace_name, value_stream.str(), units,
- important);
+ LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
+ improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
json_stream << R"("type":"scalar",)";
json_stream << R"("value":)" << value << ',';
- json_stream << R"("units":")" << units << R"("})";
+ json_stream << R"("units":")" << UnitWithDirection(units, improve_direction)
+ << R"("})";
rtc::CritScope lock(&crit_);
graphs_[graph_name].push_back(json_stream.str());
}
@@ -92,22 +95,24 @@
const double mean,
const double error,
const std::string& units,
- const bool important) {
+ const bool important,
+ webrtc::test::ImproveDirection improve_direction) {
RTC_CHECK(std::isfinite(mean));
RTC_CHECK(std::isfinite(error));
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << '{' << mean << ',' << error << '}';
- LogResultsImpl(graph_name, trace_name, value_stream.str(), units,
- important);
+ LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
+ improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
json_stream << R"("type":"list_of_scalar_values",)";
json_stream << R"("values":[)" << mean << "],";
json_stream << R"("std":)" << error << ',';
- json_stream << R"("units":")" << units << R"("})";
+ json_stream << R"("units":")" << UnitWithDirection(units, improve_direction)
+ << R"("})";
rtc::CritScope lock(&crit_);
graphs_[graph_name].push_back(json_stream.str());
}
@@ -115,7 +120,8 @@
const std::string& trace_name,
const rtc::ArrayView<const double> values,
const std::string& units,
- const bool important) {
+ const bool important,
+ webrtc::test::ImproveDirection improve_direction) {
for (double v : values) {
RTC_CHECK(std::isfinite(v));
}
@@ -125,14 +131,15 @@
value_stream << '[';
OutputListToStream(&value_stream, values);
value_stream << ']';
- LogResultsImpl(graph_name, trace_name, value_stream.str(), units,
- important);
+ LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
+ improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
json_stream << R"("type":"list_of_scalar_values",)";
json_stream << R"("values":)" << value_stream.str() << ',';
- json_stream << R"("units":")" << units << R"("})";
+ json_stream << R"("units":")" << UnitWithDirection(units, improve_direction)
+ << R"("})";
rtc::CritScope lock(&crit_);
graphs_[graph_name].push_back(json_stream.str());
}
@@ -178,7 +185,8 @@
const std::string& trace,
const std::string& values,
const std::string& units,
- bool important) {
+ bool important,
+ webrtc::test::ImproveDirection improve_direction) {
// <*>RESULT <graph_name>: <trace_name>= <value> <units>
// <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
// <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...,] <units>
@@ -188,7 +196,21 @@
fprintf(output_, "*");
}
fprintf(output_, "RESULT %s: %s= %s %s\n", graph_name.c_str(),
- trace.c_str(), values.c_str(), units.c_str());
+ trace.c_str(), values.c_str(),
+ UnitWithDirection(units, improve_direction).c_str());
+ }
+
+ std::string UnitWithDirection(
+ const std::string& units,
+ webrtc::test::ImproveDirection improve_direction) {
+ switch (improve_direction) {
+ case webrtc::test::ImproveDirection::kNone:
+ return units;
+ case webrtc::test::ImproveDirection::kSmallerIsBetter:
+ return units + "_smallerIsBetter";
+ case webrtc::test::ImproveDirection::kBiggerIsBetter:
+ return units + "_biggerIsBetter";
+ }
}
rtc::CriticalSection crit_;
@@ -254,9 +276,10 @@
const std::string& trace,
const double value,
const std::string& units,
- bool important) {
+ bool important,
+ ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResult(measurement + modifier, trace, value, units,
- important);
+ important, improve_direction);
}
void PrintResult(const std::string& measurement,
@@ -264,9 +287,10 @@
const std::string& trace,
const SamplesStatsCounter& counter,
const std::string& units,
- const bool important) {
+ const bool important,
+ ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResult(measurement + modifier, trace, counter,
- units, important);
+ units, important, improve_direction);
}
void PrintResultMeanAndError(const std::string& measurement,
@@ -275,9 +299,11 @@
const double mean,
const double error,
const std::string& units,
- bool important) {
+ bool important,
+ ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResultMeanAndError(measurement + modifier, trace,
- mean, error, units, important);
+ mean, error, units, important,
+ improve_direction);
}
void PrintResultList(const std::string& measurement,
@@ -285,9 +311,10 @@
const std::string& trace,
const rtc::ArrayView<const double> values,
const std::string& units,
- bool important) {
+ bool important,
+ ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResultList(measurement + modifier, trace, values,
- units, important);
+ units, important, improve_direction);
}
} // namespace test
diff --git a/test/testsupport/perf_test.h b/test/testsupport/perf_test.h
index 2b5b9f5..0550e44 100644
--- a/test/testsupport/perf_test.h
+++ b/test/testsupport/perf_test.h
@@ -21,6 +21,16 @@
namespace webrtc {
namespace test {
+// Metrics improver direction.
+enum class ImproveDirection {
+ // Direction is undefined.
+ kNone,
+ // Smaller value is better.
+ kSmallerIsBetter,
+ // Bigger value is better.
+ kBiggerIsBetter,
+};
+
// Prints numerical information to stdout in a controlled format, for
// post-processing. |measurement| is a description of the quantity being
// measured, e.g. "vm_peak"; |modifier| is provided as a convenience and
@@ -39,29 +49,34 @@
const std::string& trace,
const double value,
const std::string& units,
- bool important);
+ bool important,
+ ImproveDirection improve_direction = ImproveDirection::kNone);
// Like PrintResult(), but prints a (mean, standard deviation) result pair.
// The |<values>| should be two comma-separated numbers, the mean and
// standard deviation (or other error metric) of the measurement.
-void PrintResultMeanAndError(const std::string& measurement,
- const std::string& modifier,
- const std::string& trace,
- const double mean,
- const double error,
- const std::string& units,
- bool important);
+void PrintResultMeanAndError(
+ const std::string& measurement,
+ const std::string& modifier,
+ const std::string& trace,
+ const double mean,
+ const double error,
+ const std::string& units,
+ bool important,
+ ImproveDirection improve_direction = ImproveDirection::kNone);
// Like PrintResult(), but prints an entire list of results. The |values|
// will generally be a list of comma-separated numbers. A typical
// post-processing step might produce plots of their mean and standard
// deviation.
-void PrintResultList(const std::string& measurement,
- const std::string& modifier,
- const std::string& trace,
- rtc::ArrayView<const double> values,
- const std::string& units,
- bool important);
+void PrintResultList(
+ const std::string& measurement,
+ const std::string& modifier,
+ const std::string& trace,
+ rtc::ArrayView<const double> values,
+ const std::string& units,
+ bool important,
+ ImproveDirection improve_direction = ImproveDirection::kNone);
// Like PrintResult(), but prints a (mean, standard deviation) from stats
// counter. Also add specified metric to the plotable metrics output.
@@ -70,7 +85,8 @@
const std::string& trace,
const SamplesStatsCounter& counter,
const std::string& units,
- const bool important);
+ const bool important,
+ ImproveDirection improve_direction = ImproveDirection::kNone);
// Returns all perf results to date in a JSON string formatted as described in
// https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md
diff --git a/video/video_analyzer.cc b/video/video_analyzer.cc
index 2e79c24..0430faa 100644
--- a/video/video_analyzer.cc
+++ b/video/video_analyzer.cc
@@ -622,6 +622,8 @@
}
void VideoAnalyzer::PrintResults() {
+ using ::webrtc::test::ImproveDirection;
+
StopMeasuringCpuProcessTime();
int dropped_frames_diff;
{
@@ -630,24 +632,36 @@
dropped_frames_before_rendering_ + frames_.size();
}
rtc::CritScope crit(&comparison_lock_);
- PrintResult("psnr", psnr_, "dB");
- PrintResult("ssim", ssim_, "unitless");
- PrintResult("sender_time", sender_time_, "ms");
- PrintResult("receiver_time", receiver_time_, "ms");
- PrintResult("network_time", network_time_, "ms");
- PrintResult("total_delay_incl_network", end_to_end_, "ms");
- PrintResult("time_between_rendered_frames", rendered_delta_, "ms");
- PrintResult("encode_frame_rate", encode_frame_rate_, "fps");
- PrintResult("encode_time", encode_time_ms_, "ms");
- PrintResult("media_bitrate", media_bitrate_bps_, "bps");
- PrintResult("fec_bitrate", fec_bitrate_bps_, "bps");
- PrintResult("send_bandwidth", send_bandwidth_bps_, "bps");
- PrintResult("pixels_per_frame", pixels_, "count");
+ PrintResult("psnr", psnr_, "dB", ImproveDirection::kBiggerIsBetter);
+ PrintResult("ssim", ssim_, "unitless", ImproveDirection::kBiggerIsBetter);
+ PrintResult("sender_time", sender_time_, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ PrintResult("receiver_time", receiver_time_, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ PrintResult("network_time", network_time_, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ PrintResult("total_delay_incl_network", end_to_end_, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ PrintResult("time_between_rendered_frames", rendered_delta_, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ PrintResult("encode_frame_rate", encode_frame_rate_, "fps",
+ ImproveDirection::kBiggerIsBetter);
+ PrintResult("encode_time", encode_time_ms_, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ PrintResult("media_bitrate", media_bitrate_bps_, "bps",
+ ImproveDirection::kNone);
+ PrintResult("fec_bitrate", fec_bitrate_bps_, "bps", ImproveDirection::kNone);
+ PrintResult("send_bandwidth", send_bandwidth_bps_, "bps",
+ ImproveDirection::kNone);
+ PrintResult("pixels_per_frame", pixels_, "count",
+ ImproveDirection::kBiggerIsBetter);
test::PrintResult("decode_frame_rate", "", test_label_.c_str(),
- decode_frame_rate_, "fps", false);
+ decode_frame_rate_, "fps", false,
+ ImproveDirection::kBiggerIsBetter);
test::PrintResult("render_frame_rate", "", test_label_.c_str(),
- render_frame_rate_, "fps", false);
+ render_frame_rate_, "fps", false,
+ ImproveDirection::kBiggerIsBetter);
// Record the time from the last freeze until the last rendered frame to
// ensure we cover the full timespan of the session. Otherwise the metric
@@ -655,7 +669,8 @@
time_between_freezes_.AddSample(last_render_time_ - last_unfreeze_time_ms_);
// Freeze metrics.
- PrintResult("time_between_freezes", time_between_freezes_, "ms");
+ PrintResult("time_between_freezes", time_between_freezes_, "ms",
+ ImproveDirection::kBiggerIsBetter);
const double freeze_count_double = static_cast<double>(freeze_count_);
const double total_freezes_duration_ms_double =
@@ -667,7 +682,7 @@
test::PrintResult(
"freeze_duration_ratio", "", test_label_.c_str(),
total_freezes_duration_ms_double / total_frames_duration_ms_double,
- "unitless", false);
+ "unitless", false, ImproveDirection::kSmallerIsBetter);
RTC_DCHECK_LE(total_freezes_duration_ms_double,
total_frames_duration_ms_double);
@@ -677,7 +692,7 @@
if (total_frames_duration_min > 0) {
test::PrintResult("freeze_count_per_minute", "", test_label_.c_str(),
freeze_count_double / total_frames_duration_min,
- "unitless", false);
+ "unitless", false, ImproveDirection::kSmallerIsBetter);
}
}
@@ -685,35 +700,37 @@
freeze_count_double > 0
? total_freezes_duration_ms_double / freeze_count_double
: 0,
- "ms", false);
+ "ms", false, ImproveDirection::kSmallerIsBetter);
if (1000 * sum_squared_frame_durations_ > 0) {
test::PrintResult(
"harmonic_frame_rate", "", test_label_.c_str(),
total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_),
- "fps", false);
+ "fps", false, ImproveDirection::kBiggerIsBetter);
}
if (worst_frame_) {
test::PrintResult("min_psnr", "", test_label_.c_str(), worst_frame_->psnr,
- "dB", false);
+ "dB", false, ImproveDirection::kBiggerIsBetter);
}
if (receive_stream_ != nullptr) {
PrintResultWithExternalMean("decode_time", mean_decode_time_ms_,
- decode_time_ms_, "ms");
+ decode_time_ms_, "ms",
+ ImproveDirection::kSmallerIsBetter);
}
dropped_frames_ += dropped_frames_diff;
test::PrintResult("dropped_frames", "", test_label_.c_str(), dropped_frames_,
- "count", false);
+ "count", false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(),
- "%", false);
+ "%", false, ImproveDirection::kSmallerIsBetter);
#if defined(WEBRTC_WIN)
// On Linux and Mac in Resident Set some unused pages may be counted.
// Therefore this metric will depend on order in which tests are run and
// will be flaky.
- PrintResult("memory_usage", memory_usage_, "sizeInBytes");
+ PrintResult("memory_usage", memory_usage_, "sizeInBytes",
+ ImproveDirection::kSmallerIsBetter);
#endif
// Saving only the worst frame for manual analysis. Intention here is to
@@ -731,9 +748,12 @@
}
if (audio_receive_stream_ != nullptr) {
- PrintResult("audio_expand_rate", audio_expand_rate_, "unitless");
- PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless");
- PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms");
+ PrintResult("audio_expand_rate", audio_expand_rate_, "unitless",
+ ImproveDirection::kSmallerIsBetter);
+ PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless",
+ ImproveDirection::kSmallerIsBetter);
+ PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms",
+ ImproveDirection::kNone);
}
// Disable quality check for quick test, as quality checks may fail
@@ -813,18 +833,22 @@
encoded_frame_size_.AddSample(comparison.encoded_frame_size);
}
-void VideoAnalyzer::PrintResult(const char* result_type,
- Statistics stats,
- const char* unit) {
+void VideoAnalyzer::PrintResult(
+ const char* result_type,
+ Statistics stats,
+ const char* unit,
+ webrtc::test::ImproveDirection improve_direction) {
test::PrintResultMeanAndError(
result_type, "", test_label_.c_str(), stats.GetMean().value_or(0),
- stats.GetStandardDeviation().value_or(0), unit, false);
+ stats.GetStandardDeviation().value_or(0), unit, false, improve_direction);
}
-void VideoAnalyzer::PrintResultWithExternalMean(const char* result_type,
- double mean,
- Statistics stats,
- const char* unit) {
+void VideoAnalyzer::PrintResultWithExternalMean(
+ const char* result_type,
+ double mean,
+ Statistics stats,
+ const char* unit,
+ webrtc::test::ImproveDirection improve_direction) {
// If the true mean is different than the sample mean, the sample variance is
// too low. The sample variance given a known mean is obtained by adding the
// squared error between the true mean and the sample mean.
@@ -833,7 +857,8 @@
? *stats.GetVariance() + pow(mean - *stats.GetMean(), 2.0)
: 0.0;
test::PrintResultMeanAndError(result_type, "", test_label_.c_str(), mean,
- std::sqrt(compensated_variance), unit, false);
+ std::sqrt(compensated_variance), unit, false,
+ improve_direction);
}
void VideoAnalyzer::PrintSamplesToFile() {
diff --git a/video/video_analyzer.h b/video/video_analyzer.h
index 952d925..03ee87e 100644
--- a/video/video_analyzer.h
+++ b/video/video_analyzer.h
@@ -21,6 +21,7 @@
#include "rtc_base/time_utils.h"
#include "test/layer_filtering_transport.h"
#include "test/rtp_file_writer.h"
+#include "test/testsupport/perf_test.h"
namespace webrtc {
@@ -192,11 +193,16 @@
bool FrameProcessed();
void PrintResults();
void PerformFrameComparison(const FrameComparison& comparison);
- void PrintResult(const char* result_type, Statistics stats, const char* unit);
- void PrintResultWithExternalMean(const char* result_type,
- double mean,
- Statistics stats,
- const char* unit);
+ void PrintResult(const char* result_type,
+ Statistics stats,
+ const char* unit,
+ webrtc::test::ImproveDirection improve_direction);
+ void PrintResultWithExternalMean(
+ const char* result_type,
+ double mean,
+ Statistics stats,
+ const char* unit,
+ webrtc::test::ImproveDirection improve_direction);
void PrintSamplesToFile(void);
void AddCapturedFrameForComparison(const VideoFrame& video_frame);
