Migrate call_perf_tests to new perf metrics export API
Bug: b/246095034
Change-Id: I23add90d8a70ae56e3c1c8b550276344645965ee
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276625
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38202}
diff --git a/call/BUILD.gn b/call/BUILD.gn
index 36f42d8..efcb591 100644
--- a/call/BUILD.gn
+++ b/call/BUILD.gn
@@ -548,11 +548,14 @@
"../api:rtc_event_log_output_file",
"../api:simulated_network_api",
"../api/audio_codecs:builtin_audio_encoder_factory",
+ "../api/numerics",
"../api/rtc_event_log",
"../api/rtc_event_log:rtc_event_log_factory",
"../api/task_queue",
"../api/task_queue:default_task_queue_factory",
"../api/task_queue:pending_task_safety_flag",
+ "../api/test/metrics:global_metrics_logger_and_exporter",
+ "../api/test/metrics:metric",
"../api/video:builtin_video_bitrate_allocator_factory",
"../api/video:video_bitrate_allocation",
"../api/video_codecs:video_codecs_api",
@@ -584,7 +587,6 @@
"../test:field_trial",
"../test:fileutils",
"../test:null_transport",
- "../test:perf_test",
"../test:test_common",
"../test:test_support",
"../test:video_test_common",
diff --git a/call/call_perf_tests.cc b/call/call_perf_tests.cc
index 7fa7ee1..3b6db5d 100644
--- a/call/call_perf_tests.cc
+++ b/call/call_perf_tests.cc
@@ -15,9 +15,12 @@
#include "absl/strings/string_view.h"
#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/numerics/samples_stats_counter.h"
#include "api/rtc_event_log/rtc_event_log.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/task_queue/task_queue_base.h"
+#include "api/test/metrics/global_metrics_logger_and_exporter.h"
+#include "api/test/metrics/metric.h"
#include "api/test/simulated_network.h"
#include "api/video/builtin_video_bitrate_allocator_factory.h"
#include "api/video/video_bitrate_allocation.h"
@@ -49,7 +52,6 @@
#include "test/null_transport.h"
#include "test/rtp_rtcp_observer.h"
#include "test/testsupport/file_utils.h"
-#include "test/testsupport/perf_test.h"
#include "test/video_encoder_proxy_factory.h"
#include "video/transport_adapter.h"
@@ -57,9 +59,15 @@
namespace webrtc {
namespace {
+
+using ::webrtc::test::GetGlobalMetricsLogger;
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+
enum : int { // The first valid value is 1.
kTransportSequenceNumberExtensionId = 1,
};
+
} // namespace
class CallPerfTest : public test::CallTest {
@@ -133,15 +141,16 @@
if (std::abs(stats.sync_offset_ms) < kInSyncThresholdMs) {
if (first_time_in_sync_ == -1) {
first_time_in_sync_ = now_ms;
- webrtc::test::PrintResult("sync_convergence_time", test_label_,
- "synchronization", time_since_creation, "ms",
- false);
+ GetGlobalMetricsLogger()->LogSingleValueMetric(
+ "sync_convergence_time" + test_label_, "synchronization",
+ time_since_creation, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
}
if (time_since_creation > kMinRunTimeMs)
observation_complete_.Set();
}
if (first_time_in_sync_ != -1)
- sync_offset_ms_list_.push_back(stats.sync_offset_ms);
+ sync_offset_ms_list_.AddSample(stats.sync_offset_ms);
}
void set_receive_stream(VideoReceiveStreamInterface* receive_stream) {
@@ -151,8 +160,9 @@
}
void PrintResults() {
- test::PrintResultList("stream_offset", test_label_, "synchronization",
- sync_offset_ms_list_, "ms", false);
+ GetGlobalMetricsLogger()->LogMetric(
+ "stream_offset" + test_label_, "synchronization", sync_offset_ms_list_,
+ Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter);
}
private:
@@ -161,7 +171,7 @@
const int64_t creation_time_ms_;
int64_t first_time_in_sync_ = -1;
VideoReceiveStreamInterface* receive_stream_ = nullptr;
- std::vector<double> sync_offset_ms_list_;
+ SamplesStatsCounter sync_offset_ms_list_;
TaskQueueBase* const task_queue_;
};
@@ -451,7 +461,7 @@
uint32_t real_capture_timestamp = iter->second;
int time_offset_ms = real_capture_timestamp - estimated_capture_timestamp;
time_offset_ms = time_offset_ms / 90;
- time_offset_ms_list_.push_back(time_offset_ms);
+ time_offset_ms_list_.AddSample(time_offset_ms);
EXPECT_TRUE(std::abs(time_offset_ms) < threshold_ms_);
}
@@ -495,8 +505,9 @@
void PerformTest() override {
EXPECT_TRUE(Wait()) << "Timed out while waiting for estimated capture "
"NTP time to be within bounds.";
- test::PrintResultList("capture_ntp_time", "", "real - estimated",
- time_offset_ms_list_, "ms", true);
+ GetGlobalMetricsLogger()->LogMetric(
+ "capture_ntp_time", "real - estimated", time_offset_ms_list_,
+ Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter);
}
Mutex mutex_;
@@ -511,7 +522,7 @@
uint32_t rtp_start_timestamp_;
typedef std::map<uint32_t, uint32_t> FrameCaptureTimeList;
FrameCaptureTimeList capture_time_list_ RTC_GUARDED_BY(&mutex_);
- std::vector<double> time_offset_ms_list_;
+ SamplesStatsCounter time_offset_ms_list_;
} test(net_config, threshold_ms, start_time_ms, run_time_ms);
RunBaseTest(&test);
@@ -696,7 +707,7 @@
observation_complete_.Set();
}
if (converged_)
- bitrate_kbps_list_.push_back(bitrate_kbps);
+ bitrate_kbps_list_.AddSample(bitrate_kbps);
}
}));
return SEND_PACKET;
@@ -723,11 +734,12 @@
void PerformTest() override {
EXPECT_TRUE(Wait()) << "Timeout while waiting for send-bitrate stats.";
- test::PrintResultList(
- "bitrate_stats_",
- (pad_to_min_bitrate_ ? "min_transmit_bitrate"
- : "without_min_transmit_bitrate"),
- "bitrate_kbps", bitrate_kbps_list_, "kbps", false);
+ GetGlobalMetricsLogger()->LogMetric(
+ std::string("bitrate_stats_") +
+ (pad_to_min_bitrate_ ? "min_transmit_bitrate"
+ : "without_min_transmit_bitrate"),
+ "bitrate_kbps", bitrate_kbps_list_, Unit::kKilobitsPerSecond,
+ ImprovementDirection::kNeitherIsBetter);
}
VideoSendStream* send_stream_;
@@ -736,7 +748,7 @@
const int min_acceptable_bitrate_;
const int max_acceptable_bitrate_;
int num_bitrate_observations_in_range_;
- std::vector<double> bitrate_kbps_list_;
+ SamplesStatsCounter bitrate_kbps_list_;
TaskQueueBase* task_queue_;
rtc::scoped_refptr<PendingTaskSafetyFlag> task_safety_flag_;
} test(pad_to_min_bitrate, task_queue());
@@ -1003,8 +1015,9 @@
}
EXPECT_GT(last_passed_test_bitrate, -1)
<< "Minimum supported bitrate out of the test scope";
- webrtc::test::PrintResult("min_test_bitrate_", "", "min_bitrate",
- last_passed_test_bitrate, "kbps", false);
+ GetGlobalMetricsLogger()->LogSingleValueMetric(
+ "min_test_bitrate_", "min_bitrate", last_passed_test_bitrate,
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
}
void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
@@ -1130,11 +1143,14 @@
input_fps = std::max(configured_framerate.second, input_fps);
}
for (const auto& encode_frame_rate_list : encode_frame_rate_lists_) {
- const std::vector<double>& values = encode_frame_rate_list.second;
- test::PrintResultList("substream", "", "encode_frame_rate", values,
- "fps", false);
- double average_fps =
- std::accumulate(values.begin(), values.end(), 0.0) / values.size();
+ const SamplesStatsCounter& values = encode_frame_rate_list.second;
+ GetGlobalMetricsLogger()->LogMetric(
+ "substream_fps", "encode_frame_rate", values, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
+ if (values.IsEmpty()) {
+ continue;
+ }
+ double average_fps = values.GetAverage();
uint32_t ssrc = encode_frame_rate_list.first;
double expected_fps = configured_framerates_.find(ssrc)->second;
if (expected_fps != input_fps)
@@ -1149,7 +1165,7 @@
task_queue_->PostTask([this, now]() {
VideoSendStream::Stats stats = send_stream_->GetStats();
for (const auto& stat : stats.substreams) {
- encode_frame_rate_lists_[stat.first].push_back(
+ encode_frame_rate_lists_[stat.first].AddSample(
stat.second.encode_frame_rate);
}
if (now - start_time_ > kMinRunTime) {
@@ -1169,7 +1185,7 @@
const Timestamp start_time_;
Timestamp last_getstats_time_;
VideoSendStream* send_stream_;
- std::map<uint32_t, std::vector<double>> encode_frame_rate_lists_;
+ std::map<uint32_t, SamplesStatsCounter> encode_frame_rate_lists_;
std::map<uint32_t, double> configured_framerates_;
} test(encoder_factory, payload_name, max_framerates, task_queue());
diff --git a/call/rampup_tests.cc b/call/rampup_tests.cc
index 2c47243..eea54ed 100644
--- a/call/rampup_tests.cc
+++ b/call/rampup_tests.cc
@@ -19,6 +19,8 @@
#include "api/task_queue/default_task_queue_factory.h"
#include "api/task_queue/task_queue_base.h"
#include "api/task_queue/task_queue_factory.h"
+#include "api/test/metrics/global_metrics_logger_and_exporter.h"
+#include "api/test/metrics/metric.h"
#include "call/fake_network_pipe.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
@@ -28,7 +30,6 @@
#include "rtc_base/time_utils.h"
#include "test/encoder_settings.h"
#include "test/gtest.h"
-#include "test/testsupport/perf_test.h"
ABSL_FLAG(std::string,
ramp_dump_name,
@@ -38,6 +39,10 @@
namespace webrtc {
namespace {
+using ::webrtc::test::GetGlobalMetricsLogger;
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+
constexpr TimeDelta kPollInterval = TimeDelta::Millis(20);
static const int kExpectedHighVideoBitrateBps = 80000;
static const int kExpectedHighAudioBitrateBps = 30000;
@@ -52,6 +57,7 @@
ssrcs.push_back(static_cast<uint32_t>(ssrc_offset + i));
return ssrcs;
}
+
} // namespace
RampUpTester::RampUpTester(size_t num_video_streams,
@@ -332,12 +338,12 @@
void RampUpTester::ReportResult(
absl::string_view measurement,
size_t value,
- absl::string_view units,
- test::ImproveDirection improve_direction) const {
- webrtc::test::PrintResult(
- measurement, "",
+ Unit unit,
+ ImprovementDirection improvement_direction) const {
+ GetGlobalMetricsLogger()->LogSingleValueMetric(
+ measurement,
::testing::UnitTest::GetInstance()->current_test_info()->name(), value,
- units, false);
+ unit, improvement_direction);
}
void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream,
@@ -392,21 +398,21 @@
}
if (report_perf_stats_) {
- ReportResult("ramp-up-media-sent", media_sent, "bytes",
- test::ImproveDirection::kBiggerIsBetter);
- ReportResult("ramp-up-padding-sent", padding_sent, "bytes",
- test::ImproveDirection::kSmallerIsBetter);
- ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, "bytes",
- test::ImproveDirection::kBiggerIsBetter);
- ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, "bytes",
- test::ImproveDirection::kSmallerIsBetter);
+ ReportResult("ramp-up-media-sent", media_sent, Unit::kBytes,
+ ImprovementDirection::kBiggerIsBetter);
+ ReportResult("ramp-up-padding-sent", padding_sent, Unit::kBytes,
+ ImprovementDirection::kSmallerIsBetter);
+ ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, Unit::kBytes,
+ ImprovementDirection::kBiggerIsBetter);
+ ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, Unit::kBytes,
+ ImprovementDirection::kSmallerIsBetter);
if (ramp_up_finished_ms_ >= 0) {
ReportResult("ramp-up-time", ramp_up_finished_ms_ - test_start_ms_,
- "milliseconds", test::ImproveDirection::kSmallerIsBetter);
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
}
ReportResult("ramp-up-average-network-latency",
- send_transport_->GetAverageDelayMs(), "milliseconds",
- test::ImproveDirection::kSmallerIsBetter);
+ send_transport_->GetAverageDelayMs(), Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
}
}
@@ -531,10 +537,10 @@
EXPECT_FALSE(suspended);
if (bitrate_bps >= GetExpectedHighBitrate()) {
if (report_perf_stats_) {
- webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
- "first_rampup", now - state_start_ms_, "ms",
- false,
- test::ImproveDirection::kSmallerIsBetter);
+ GetGlobalMetricsLogger()->LogSingleValueMetric(
+ "ramp_up_down_up" + GetModifierString(), "first_rampup",
+ now - state_start_ms_, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
}
// Apply loss during the transition between states if FEC is enabled.
forward_transport_config_.loss_percent = loss_rates_[test_state_];
@@ -548,10 +554,10 @@
if (bitrate_bps < kLowBandwidthLimitBps + kLowBitrateMarginBps &&
suspended == check_suspend_state) {
if (report_perf_stats_) {
- webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
- "rampdown", now - state_start_ms_, "ms",
- false,
- test::ImproveDirection::kSmallerIsBetter);
+ GetGlobalMetricsLogger()->LogSingleValueMetric(
+ "ramp_up_down_up" + GetModifierString(), "rampdown",
+ now - state_start_ms_, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
}
// Apply loss during the transition between states if FEC is enabled.
forward_transport_config_.loss_percent = loss_rates_[test_state_];
@@ -563,13 +569,14 @@
case kSecondRampup:
if (bitrate_bps >= GetExpectedHighBitrate() && !suspended) {
if (report_perf_stats_) {
- webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
- "second_rampup", now - state_start_ms_,
- "ms", false,
- test::ImproveDirection::kSmallerIsBetter);
+ GetGlobalMetricsLogger()->LogSingleValueMetric(
+ "ramp_up_down_up" + GetModifierString(), "second_rampup",
+ now - state_start_ms_, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
ReportResult("ramp-up-down-up-average-network-latency",
- send_transport_->GetAverageDelayMs(), "milliseconds",
- test::ImproveDirection::kSmallerIsBetter);
+ send_transport_->GetAverageDelayMs(),
+ Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
}
// Apply loss during the transition between states if FEC is enabled.
forward_transport_config_.loss_percent = loss_rates_[test_state_];
@@ -729,4 +736,5 @@
false, task_queue());
RunBaseTest(&test);
}
+
} // namespace webrtc
diff --git a/call/rampup_tests.h b/call/rampup_tests.h
index 4e9217f..de0022c 100644
--- a/call/rampup_tests.h
+++ b/call/rampup_tests.h
@@ -20,13 +20,13 @@
#include "absl/strings/string_view.h"
#include "api/rtc_event_log/rtc_event_log.h"
#include "api/task_queue/task_queue_base.h"
+#include "api/test/metrics/metric.h"
#include "api/test/simulated_network.h"
#include "call/call.h"
#include "call/simulated_network.h"
#include "rtc_base/event.h"
#include "rtc_base/task_utils/repeating_task.h"
#include "test/call_test.h"
-#include "test/testsupport/perf_test.h"
namespace webrtc {
@@ -68,8 +68,8 @@
void ReportResult(absl::string_view measurement,
size_t value,
- absl::string_view units,
- test::ImproveDirection improve_direction) const;
+ test::Unit unit,
+ test::ImprovementDirection improvement_direction) const;
void TriggerTestDone();
Clock* const clock_;
@@ -166,5 +166,6 @@
int sent_bytes_;
std::vector<int> loss_rates_;
};
+
} // namespace webrtc
#endif // CALL_RAMPUP_TESTS_H_
