Extract BWE stats collection.
Extract collection of BWE stats from DefaultVideoQualityAnalyzer to
separate class to prepare for migration on new GetStats API and simplify
quality analyzer.
Bug: webrtc:11381
Change-Id: I0e7e2d7e40b467d7a42633a72a7ffc49ebcb0237
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169444
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30650}
diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn
index 3855ad0..dd4ef21 100644
--- a/test/pc/e2e/BUILD.gn
+++ b/test/pc/e2e/BUILD.gn
@@ -272,6 +272,7 @@
":stats_poller",
":test_peer",
":video_quality_analyzer_injection_helper",
+ ":video_quality_metrics_reporter",
"../..:field_trial",
"../..:platform_video_capturer",
"../..:video_test_common",
@@ -466,6 +467,22 @@
]
}
+rtc_library("video_quality_metrics_reporter") {
+ visibility = [ "*" ]
+
+ testonly = true
+ sources = [
+ "analyzer/video/video_quality_metrics_reporter.cc",
+ "analyzer/video/video_quality_metrics_reporter.h",
+ ]
+ deps = [
+ "../..:perf_test",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../rtc_base:criticalsection",
+ "../../../rtc_base:rtc_numerics",
+ ]
+}
+
rtc_library("default_video_quality_analyzer") {
visibility = [ "*" ]
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
index 0b0c3b1..a1c5d0a 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -26,7 +26,6 @@
constexpr int kMaxActiveComparisons = 10;
constexpr int kFreezeThresholdMs = 150;
constexpr int kMicrosPerSecond = 1000000;
-constexpr int kBitsInByte = 8;
void LogFrameCounters(const std::string& name, const FrameCounters& counters) {
RTC_LOG(INFO) << "[" << name << "] Captured : " << counters.captured;
@@ -422,59 +421,6 @@
return analyzer_stats_;
}
-// TODO(bugs.webrtc.org/10430): Migrate to the new GetStats as soon as
-// bugs.webrtc.org/10428 is fixed.
-void DefaultVideoQualityAnalyzer::OnStatsReports(
- const std::string& pc_label,
- const StatsReports& stats_reports) {
- for (const StatsReport* stats_report : stats_reports) {
- // The only stats collected by this analyzer are present in
- // kStatsReportTypeBwe reports, so all other reports are just ignored.
- if (stats_report->type() != StatsReport::StatsType::kStatsReportTypeBwe) {
- continue;
- }
- const webrtc::StatsReport::Value* available_send_bandwidth =
- stats_report->FindValue(
- StatsReport::StatsValueName::kStatsValueNameAvailableSendBandwidth);
- const webrtc::StatsReport::Value* retransmission_bitrate =
- stats_report->FindValue(
- StatsReport::StatsValueName::kStatsValueNameRetransmitBitrate);
- const webrtc::StatsReport::Value* transmission_bitrate =
- stats_report->FindValue(
- StatsReport::StatsValueName::kStatsValueNameTransmitBitrate);
- const webrtc::StatsReport::Value* actual_encode_bitrate =
- stats_report->FindValue(
- StatsReport::StatsValueName::kStatsValueNameActualEncBitrate);
- const webrtc::StatsReport::Value* target_encode_bitrate =
- stats_report->FindValue(
- StatsReport::StatsValueName::kStatsValueNameTargetEncBitrate);
- RTC_CHECK(available_send_bandwidth);
- RTC_CHECK(retransmission_bitrate);
- RTC_CHECK(transmission_bitrate);
- RTC_CHECK(actual_encode_bitrate);
- RTC_CHECK(target_encode_bitrate);
-
- rtc::CritScope crit(&video_bwe_stats_lock_);
- VideoBweStats& video_bwe_stats = video_bwe_stats_[pc_label];
- video_bwe_stats.available_send_bandwidth.AddSample(
- available_send_bandwidth->int_val());
- video_bwe_stats.transmission_bitrate.AddSample(
- transmission_bitrate->int_val());
- video_bwe_stats.retransmission_bitrate.AddSample(
- retransmission_bitrate->int_val());
- video_bwe_stats.actual_encode_bitrate.AddSample(
- actual_encode_bitrate->int_val());
- video_bwe_stats.target_encode_bitrate.AddSample(
- target_encode_bitrate->int_val());
- }
-}
-
-std::map<std::string, VideoBweStats>
-DefaultVideoQualityAnalyzer::GetVideoBweStats() const {
- rtc::CritScope crit(&video_bwe_stats_lock_);
- return video_bwe_stats_;
-}
-
void DefaultVideoQualityAnalyzer::AddComparison(
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
@@ -620,12 +566,6 @@
ReportResults(GetTestCaseName(item.first), item.second,
stream_frame_counters_.at(item.first));
}
- {
- rtc::CritScope video_bwe_crit(&video_bwe_stats_lock_);
- for (const auto& item : video_bwe_stats_) {
- ReportVideoBweResults(GetTestCaseName(item.first), item.second);
- }
- }
LogFrameCounters("Global", frame_counters_);
for (auto& item : stream_stats_) {
LogFrameCounters(item.first, stream_frame_counters_.at(item.first));
@@ -645,26 +585,6 @@
<< analyzer_stats_.memory_overloaded_comparisons_done;
}
-void DefaultVideoQualityAnalyzer::ReportVideoBweResults(
- const std::string& test_case_name,
- const VideoBweStats& video_bwe_stats) {
- ReportResult("available_send_bandwidth", test_case_name,
- video_bwe_stats.available_send_bandwidth / kBitsInByte,
- "bytesPerSecond");
- ReportResult("transmission_bitrate", test_case_name,
- video_bwe_stats.transmission_bitrate / kBitsInByte,
- "bytesPerSecond");
- ReportResult("retransmission_bitrate", test_case_name,
- video_bwe_stats.retransmission_bitrate / kBitsInByte,
- "bytesPerSecond");
- ReportResult("actual_encode_bitrate", test_case_name,
- video_bwe_stats.actual_encode_bitrate / kBitsInByte,
- "bytesPerSecond");
- ReportResult("target_encode_bitrate", test_case_name,
- video_bwe_stats.target_encode_bitrate / kBitsInByte,
- "bytesPerSecond");
-}
-
void DefaultVideoQualityAnalyzer::ReportResults(
const std::string& test_case_name,
const StreamStats& stats,
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
index 3ed7a65..219a77b 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
@@ -124,14 +124,6 @@
int64_t memory_overloaded_comparisons_done = 0;
};
-struct VideoBweStats {
- SamplesStatsCounter available_send_bandwidth;
- SamplesStatsCounter transmission_bitrate;
- SamplesStatsCounter retransmission_bitrate;
- SamplesStatsCounter actual_encode_bitrate;
- SamplesStatsCounter target_encode_bitrate;
-};
-
class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public:
explicit DefaultVideoQualityAnalyzer(
@@ -157,6 +149,8 @@
void OnDecoderError(uint16_t frame_id, int32_t error_code) override;
void Stop() override;
std::string GetStreamLabel(uint16_t frame_id) override;
+ void OnStatsReports(const std::string& pc_label,
+ const StatsReports& stats_reports) override {}
// Returns set of stream labels, that were met during test call.
std::set<std::string> GetKnownVideoStreams() const;
@@ -169,13 +163,6 @@
std::map<std::string, StreamStats> GetStats() const;
AnalyzerStats GetAnalyzerStats() const;
- // Will be called everytime new stats reports are available for the
- // Peer Connection identified by |pc_label|.
- void OnStatsReports(const std::string& pc_label,
- const StatsReports& stats_reports) override;
-
- std::map<std::string, VideoBweStats> GetVideoBweStats() const;
-
private:
struct FrameStats {
FrameStats(std::string stream_label, Timestamp captured_time);
@@ -285,8 +272,6 @@
void ProcessComparison(const FrameComparison& comparison);
// Report results for all metrics for all streams.
void ReportResults();
- static void ReportVideoBweResults(const std::string& test_case_name,
- const VideoBweStats& video_bwe_stats);
void ReportResults(const std::string& test_case_name,
const StreamStats& stats,
const FrameCounters& frame_counters)
@@ -344,12 +329,6 @@
std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(comparison_lock_);
- rtc::CriticalSection video_bwe_stats_lock_;
- // Map between a peer connection label (provided by the framework) and
- // its video BWE stats.
- std::map<std::string, VideoBweStats> video_bwe_stats_
- RTC_GUARDED_BY(video_bwe_stats_lock_);
-
std::vector<std::unique_ptr<rtc::PlatformThread>> thread_pool_;
rtc::Event comparison_available_event_;
};
diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
new file mode 100644
index 0000000..2c7eb0e
--- /dev/null
+++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+constexpr int kBitsInByte = 8;
+
+} // namespace
+
+void VideoQualityMetricsReporter::Start(absl::string_view test_case_name) {
+ test_case_name_ = std::string(test_case_name);
+}
+
+// TODO(bugs.webrtc.org/10430): Migrate to the new GetStats as soon as
+// bugs.webrtc.org/10428 is fixed.
+void VideoQualityMetricsReporter::OnStatsReports(
+ const std::string& pc_label,
+ const StatsReports& stats_reports) {
+ for (const StatsReport* stats_report : stats_reports) {
+ // The only stats collected by this analyzer are present in
+ // kStatsReportTypeBwe reports, so all other reports are just ignored.
+ if (stats_report->type() != StatsReport::StatsType::kStatsReportTypeBwe) {
+ continue;
+ }
+ const webrtc::StatsReport::Value* available_send_bandwidth =
+ stats_report->FindValue(
+ StatsReport::StatsValueName::kStatsValueNameAvailableSendBandwidth);
+ const webrtc::StatsReport::Value* retransmission_bitrate =
+ stats_report->FindValue(
+ StatsReport::StatsValueName::kStatsValueNameRetransmitBitrate);
+ const webrtc::StatsReport::Value* transmission_bitrate =
+ stats_report->FindValue(
+ StatsReport::StatsValueName::kStatsValueNameTransmitBitrate);
+ const webrtc::StatsReport::Value* actual_encode_bitrate =
+ stats_report->FindValue(
+ StatsReport::StatsValueName::kStatsValueNameActualEncBitrate);
+ const webrtc::StatsReport::Value* target_encode_bitrate =
+ stats_report->FindValue(
+ StatsReport::StatsValueName::kStatsValueNameTargetEncBitrate);
+ RTC_CHECK(available_send_bandwidth);
+ RTC_CHECK(retransmission_bitrate);
+ RTC_CHECK(transmission_bitrate);
+ RTC_CHECK(actual_encode_bitrate);
+ RTC_CHECK(target_encode_bitrate);
+
+ rtc::CritScope crit(&video_bwe_stats_lock_);
+ VideoBweStats& video_bwe_stats = video_bwe_stats_[pc_label];
+ video_bwe_stats.available_send_bandwidth.AddSample(
+ available_send_bandwidth->int_val());
+ video_bwe_stats.transmission_bitrate.AddSample(
+ transmission_bitrate->int_val());
+ video_bwe_stats.retransmission_bitrate.AddSample(
+ retransmission_bitrate->int_val());
+ video_bwe_stats.actual_encode_bitrate.AddSample(
+ actual_encode_bitrate->int_val());
+ video_bwe_stats.target_encode_bitrate.AddSample(
+ target_encode_bitrate->int_val());
+ }
+}
+
+void VideoQualityMetricsReporter::StopAndReportResults() {
+ rtc::CritScope video_bwe_crit(&video_bwe_stats_lock_);
+ for (const auto& item : video_bwe_stats_) {
+ ReportVideoBweResults(GetTestCaseName(item.first), item.second);
+ }
+}
+
+std::string VideoQualityMetricsReporter::GetTestCaseName(
+ const std::string& stream_label) const {
+ return test_case_name_ + "/" + stream_label;
+}
+
+void VideoQualityMetricsReporter::ReportVideoBweResults(
+ const std::string& test_case_name,
+ const VideoBweStats& video_bwe_stats) {
+ ReportResult("available_send_bandwidth", test_case_name,
+ video_bwe_stats.available_send_bandwidth / kBitsInByte,
+ "bytesPerSecond");
+ ReportResult("transmission_bitrate", test_case_name,
+ video_bwe_stats.transmission_bitrate / kBitsInByte,
+ "bytesPerSecond");
+ ReportResult("retransmission_bitrate", test_case_name,
+ video_bwe_stats.retransmission_bitrate / kBitsInByte,
+ "bytesPerSecond");
+ ReportResult("actual_encode_bitrate", test_case_name,
+ video_bwe_stats.actual_encode_bitrate / kBitsInByte,
+ "bytesPerSecond");
+ ReportResult("target_encode_bitrate", test_case_name,
+ video_bwe_stats.target_encode_bitrate / kBitsInByte,
+ "bytesPerSecond");
+}
+
+void VideoQualityMetricsReporter::ReportResult(
+ const std::string& metric_name,
+ const std::string& test_case_name,
+ const SamplesStatsCounter& counter,
+ const std::string& unit,
+ webrtc::test::ImproveDirection improve_direction) {
+ test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
+ /*important=*/false, improve_direction);
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
new file mode 100644
index 0000000..28cb0d8
--- /dev/null
+++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_
+
+#include <map>
+#include <string>
+
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "rtc_base/critical_section.h"
+#include "rtc_base/numerics/samples_stats_counter.h"
+#include "test/testsupport/perf_test.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+struct VideoBweStats {
+ SamplesStatsCounter available_send_bandwidth;
+ SamplesStatsCounter transmission_bitrate;
+ SamplesStatsCounter retransmission_bitrate;
+ SamplesStatsCounter actual_encode_bitrate;
+ SamplesStatsCounter target_encode_bitrate;
+};
+
+class VideoQualityMetricsReporter
+ : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
+ public:
+ VideoQualityMetricsReporter() = default;
+ ~VideoQualityMetricsReporter() override = default;
+
+ void Start(absl::string_view test_case_name) override;
+ void OnStatsReports(const std::string& pc_label,
+ const StatsReports& reports) override;
+ void StopAndReportResults() override;
+
+ private:
+ std::string GetTestCaseName(const std::string& stream_label) const;
+ static void ReportVideoBweResults(const std::string& test_case_name,
+ const VideoBweStats& video_bwe_stats);
+ // Report result for single metric for specified stream.
+ static void ReportResult(const std::string& metric_name,
+ const std::string& test_case_name,
+ const SamplesStatsCounter& counter,
+ const std::string& unit,
+ webrtc::test::ImproveDirection improve_direction =
+ webrtc::test::ImproveDirection::kNone);
+
+ std::string test_case_name_;
+
+ rtc::CriticalSection video_bwe_stats_lock_;
+ // Map between a peer connection label (provided by the framework) and
+ // its video BWE stats.
+ std::map<std::string, VideoBweStats> video_bwe_stats_
+ RTC_GUARDED_BY(video_bwe_stats_lock_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_
diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc
index faf1aaa..f16389f 100644
--- a/test/pc/e2e/peer_connection_quality_test.cc
+++ b/test/pc/e2e/peer_connection_quality_test.cc
@@ -35,6 +35,7 @@
#include "test/frame_generator_capturer.h"
#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h"
#include "test/pc/e2e/stats_poller.h"
#include "test/platform_video_capturer.h"
#include "test/testsupport/file_utils.h"
@@ -321,6 +322,8 @@
video_analyzer_threads =
std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads);
RTC_LOG(INFO) << "video_analyzer_threads=" << video_analyzer_threads;
+ quality_metrics_reporters_.push_back(
+ std::make_unique<VideoQualityMetricsReporter>());
video_quality_analyzer_injection_helper_->Start(test_case_name_,
video_analyzer_threads);
