Introduce QualityMetricsReporter and implement network stats gathering
QualityMetricsReporter helps to keep network emulation framework and
peer connection level test framework separated. Also it provides
ability to gather statistics from any component around with
correlation with call start and end.
Bug: webrtc:10138
Change-Id: Ib3330a8d35481fde77fcf77d2271d6cfcf188fec
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132718
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Peter Slatala <psla@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27759}
diff --git a/api/test/peerconnection_quality_test_fixture.h b/api/test/peerconnection_quality_test_fixture.h
index 967a176..66c056e 100644
--- a/api/test/peerconnection_quality_test_fixture.h
+++ b/api/test/peerconnection_quality_test_fixture.h
@@ -193,6 +193,20 @@
double video_encoder_bitrate_multiplier = 1.0;
};
+ // Represent an entity that will report quality metrics after test.
+ class QualityMetricsReporter {
+ public:
+ virtual ~QualityMetricsReporter() = default;
+
+ // Invoked by framework after peer connection factory and peer connection
+ // itself will be created but before offer/answer exchange will be started.
+ virtual void Start(absl::string_view test_case_name) = 0;
+
+ // Invoked by framework after call is ended and peer connection factory and
+ // peer connection are destroyed.
+ virtual void StopAndReportResults() = 0;
+ };
+
virtual ~PeerConnectionE2EQualityTestFixture() = default;
// Add activity that will be executed on the best effort at least after
@@ -209,6 +223,10 @@
TimeDelta interval,
std::function<void(TimeDelta)> func) = 0;
+ // Add stats reporter entity to observe the test.
+ virtual void AddQualityMetricsReporter(
+ std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) = 0;
+
// Add a new peer to the call and return an object through which caller
// can configure peer's behavior.
// |network_thread| will be used as network thread for peer's peer connection
diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn
index 43d6ede..1f398f2 100644
--- a/test/pc/e2e/BUILD.gn
+++ b/test/pc/e2e/BUILD.gn
@@ -312,6 +312,7 @@
deps = [
":default_audio_quality_analyzer",
":default_video_quality_analyzer",
+ ":network_quality_metrics_reporter",
"../../../api:callfactory_api",
"../../../api:create_network_emulation_manager",
"../../../api:create_peerconnection_quality_test_fixture",
@@ -436,3 +437,18 @@
"//third_party/abseil-cpp/absl/memory",
]
}
+
+rtc_source_set("network_quality_metrics_reporter") {
+ visibility = [ "*" ]
+ testonly = true
+ sources = [
+ "network_quality_metrics_reporter.cc",
+ "network_quality_metrics_reporter.h",
+ ]
+ deps = [
+ "../..:perf_test",
+ "../../../api:network_emulation_manager_api",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../rtc_base:rtc_event",
+ ]
+}
diff --git a/test/pc/e2e/network_quality_metrics_reporter.cc b/test/pc/e2e/network_quality_metrics_reporter.cc
new file mode 100644
index 0000000..70927fe
--- /dev/null
+++ b/test/pc/e2e/network_quality_metrics_reporter.cc
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/network_quality_metrics_reporter.h"
+
+#include <utility>
+
+#include "rtc_base/event.h"
+#include "test/testsupport/perf_test.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+constexpr int kStatsWaitTimeoutMs = 1000;
+
+}
+
+void NetworkQualityMetricsReporter::Start(absl::string_view test_case_name) {
+ test_case_name_ = std::string(test_case_name);
+ // Check that network stats are clean before test execution.
+ EmulatedNetworkStats alice_stats = PopulateStats(alice_network_);
+ RTC_CHECK_EQ(alice_stats.packets_sent, 0);
+ RTC_CHECK_EQ(alice_stats.packets_received, 0);
+ EmulatedNetworkStats bob_stats = PopulateStats(bob_network_);
+ RTC_CHECK_EQ(bob_stats.packets_sent, 0);
+ RTC_CHECK_EQ(bob_stats.packets_received, 0);
+}
+
+void NetworkQualityMetricsReporter::StopAndReportResults() {
+ EmulatedNetworkStats alice_stats = PopulateStats(alice_network_);
+ EmulatedNetworkStats bob_stats = PopulateStats(bob_network_);
+ ReportStats("alice", alice_stats,
+ alice_stats.packets_sent - bob_stats.packets_received);
+ ReportStats("bob", bob_stats,
+ bob_stats.packets_sent - alice_stats.packets_received);
+}
+
+EmulatedNetworkStats NetworkQualityMetricsReporter::PopulateStats(
+ EmulatedNetworkManagerInterface* network) {
+ rtc::Event wait;
+ EmulatedNetworkStats stats;
+ network->GetStats([&](const EmulatedNetworkStats& s) {
+ stats = s;
+ wait.Set();
+ });
+ bool stats_received = wait.Wait(kStatsWaitTimeoutMs);
+ RTC_CHECK(stats_received);
+ return stats;
+}
+
+void NetworkQualityMetricsReporter::ReportStats(
+ const std::string& network_label,
+ const EmulatedNetworkStats& stats,
+ int64_t packet_loss) {
+ ReportResult("bytes_sent", network_label, stats.bytes_sent.bytes(),
+ "sizeInBytes");
+ ReportResult("packets_sent", network_label, stats.packets_sent, "unitless");
+ ReportResult("average_send_rate", network_label,
+ stats.AverageSendRate().bytes_per_sec(), "bytesPerSecond");
+ ReportResult("bytes_dropped", network_label, stats.bytes_dropped.bytes(),
+ "sizeInBytes");
+ ReportResult("packets_dropped", network_label, stats.packets_dropped,
+ "unitless");
+ ReportResult("bytes_received", network_label, stats.bytes_received.bytes(),
+ "sizeInBytes");
+ ReportResult("packets_received", network_label, stats.packets_received,
+ "unitless");
+ ReportResult("average_receive_rate", network_label,
+ stats.AverageReceiveRate().bytes_per_sec(), "bytesPerSecond");
+ ReportResult("sent_packets_loss", network_label, packet_loss, "unitless");
+}
+
+void NetworkQualityMetricsReporter::ReportResult(
+ const std::string& metric_name,
+ const std::string& network_label,
+ const double value,
+ const std::string& unit) const {
+ test::PrintResult(metric_name, /*modifier=*/"",
+ GetTestCaseName(network_label), value, unit,
+ /*important=*/false);
+}
+
+std::string NetworkQualityMetricsReporter::GetTestCaseName(
+ const std::string& network_label) const {
+ return test_case_name_ + "/" + network_label;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/test/pc/e2e/network_quality_metrics_reporter.h b/test/pc/e2e/network_quality_metrics_reporter.h
new file mode 100644
index 0000000..52106ef
--- /dev/null
+++ b/test/pc/e2e/network_quality_metrics_reporter.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_
+#define TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_
+
+#include <string>
+
+#include "api/test/network_emulation_manager.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class NetworkQualityMetricsReporter
+ : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
+ public:
+ NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
+ EmulatedNetworkManagerInterface* bob_network)
+ : alice_network_(alice_network), bob_network_(bob_network) {}
+ ~NetworkQualityMetricsReporter() override = default;
+
+ // Network stats must be empty when this method will be invoked.
+ void Start(absl::string_view test_case_name) override;
+ void StopAndReportResults() override;
+
+ private:
+ static EmulatedNetworkStats PopulateStats(
+ EmulatedNetworkManagerInterface* network);
+ void ReportStats(const std::string& network_label,
+ const EmulatedNetworkStats& stats,
+ int64_t packet_loss);
+ void ReportResult(const std::string& metric_name,
+ const std::string& network_label,
+ const double value,
+ const std::string& unit) const;
+ std::string GetTestCaseName(const std::string& network_label) const;
+
+ std::string test_case_name_;
+
+ EmulatedNetworkManagerInterface* alice_network_;
+ EmulatedNetworkManagerInterface* bob_network_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_
diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc
index 62e0d54..33e666b 100644
--- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc
+++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc
@@ -20,6 +20,7 @@
#include "test/gtest.h"
#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+#include "test/pc/e2e/network_quality_metrics_reporter.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
@@ -113,6 +114,10 @@
bob->SetAudioConfig(std::move(audio_config));
});
+ fixture->AddQualityMetricsReporter(
+ absl::make_unique<NetworkQualityMetricsReporter>(alice_network,
+ bob_network));
+
RunParams run_params(TimeDelta::seconds(7));
run_params.video_encoder_bitrate_multiplier = 1.1;
fixture->Run(run_params);
diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc
index 7f3aafd..7fa53fa 100644
--- a/test/pc/e2e/peer_connection_quality_test.cc
+++ b/test/pc/e2e/peer_connection_quality_test.cc
@@ -197,6 +197,11 @@
remaining_delay.ms());
}
+void PeerConnectionE2EQualityTest::AddQualityMetricsReporter(
+ std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) {
+ quality_metrics_reporters_.push_back(std::move(quality_metrics_reporter));
+}
+
void PeerConnectionE2EQualityTest::AddPeer(
rtc::Thread* network_thread,
rtc::NetworkManager* network_manager,
@@ -292,6 +297,9 @@
video_quality_analyzer_injection_helper_->Start(test_case_name_,
video_analyzer_threads);
audio_quality_analyzer_->Start(test_case_name_, &analyzer_helper_);
+ for (auto& reporter : quality_metrics_reporters_) {
+ reporter->Start(test_case_name_);
+ }
// Start RTCEventLog recording if requested.
if (alice_->params()->rtc_event_log_path) {
@@ -364,6 +372,9 @@
audio_quality_analyzer_->Stop();
video_quality_analyzer_injection_helper_->Stop();
+ for (auto& reporter : quality_metrics_reporters_) {
+ reporter->StopAndReportResults();
+ }
// Ensuring that TestPeers have been destroyed in order to correctly close
// Audio dumps.
diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h
index e827921..93c7508 100644
--- a/test/pc/e2e/peer_connection_quality_test.h
+++ b/test/pc/e2e/peer_connection_quality_test.h
@@ -155,6 +155,8 @@
using RunParams = PeerConnectionE2EQualityTestFixture::RunParams;
using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer;
+ using QualityMetricsReporter =
+ PeerConnectionE2EQualityTestFixture::QualityMetricsReporter;
PeerConnectionE2EQualityTest(
std::string test_case_name,
@@ -169,6 +171,9 @@
TimeDelta interval,
std::function<void(TimeDelta)> func) override;
+ void AddQualityMetricsReporter(std::unique_ptr<QualityMetricsReporter>
+ quality_metrics_reporter) override;
+
void AddPeer(rtc::Thread* network_thread,
rtc::NetworkManager* network_manager,
rtc::FunctionView<void(PeerConfigurer*)> configurer) override;
@@ -232,6 +237,8 @@
std::unique_ptr<TestPeer> alice_;
std::unique_ptr<TestPeer> bob_;
+ std::vector<std::unique_ptr<QualityMetricsReporter>>
+ quality_metrics_reporters_;
std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
alice_video_sources_;
