Refactor PC level smoke test for adding new network stats reporter
New network stats reporter will require peer name to be passed into
constructor. Because peer name can be set inside PeerConfigurer adding
of the reporter have to be moved to PeerConfigurer creation.
Bug: webrtc:11756
Change-Id: I4840566a3cd2c73fbb1d186aa1c40ebf889fd830
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/180805
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Andrey Logvin <landrey@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31841}
diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc
index b09b093..39dbbf6 100644
--- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc
+++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc
@@ -46,74 +46,58 @@
using EchoEmulationConfig =
PeerConnectionE2EQualityTestFixture::EchoEmulationConfig;
- void RunTest(const std::string& test_case_name,
- const RunParams& run_params,
- rtc::FunctionView<void(PeerConfigurer*)> alice_configurer,
- rtc::FunctionView<void(PeerConfigurer*)> bob_configurer) {
- // Setup emulated network
- std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
- CreateNetworkEmulationManager();
-
- auto alice_network_behavior =
- std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig());
- SimulatedNetwork* alice_network_behavior_ptr = alice_network_behavior.get();
- EmulatedNetworkNode* alice_node =
- network_emulation_manager->CreateEmulatedNode(
- std::move(alice_network_behavior));
- EmulatedNetworkNode* bob_node =
- network_emulation_manager->CreateEmulatedNode(
- std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
- auto* alice_endpoint =
- network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
- EmulatedEndpoint* bob_endpoint =
- network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
- network_emulation_manager->CreateRoute(alice_endpoint, {alice_node},
- bob_endpoint);
- network_emulation_manager->CreateRoute(bob_endpoint, {bob_node},
- alice_endpoint);
-
- // Create analyzers.
- std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer =
- std::make_unique<DefaultVideoQualityAnalyzer>(
- network_emulation_manager->time_controller()->GetClock());
- // This is only done for the sake of smoke testing. In general there should
- // be no need to explicitly pull data from analyzers after the run.
- auto* video_analyzer_ptr =
- static_cast<DefaultVideoQualityAnalyzer*>(video_quality_analyzer.get());
-
- auto fixture = CreatePeerConnectionE2EQualityTestFixture(
- test_case_name, *network_emulation_manager->time_controller(),
+ void SetUp() override {
+ network_emulation_ = CreateNetworkEmulationManager();
+ auto video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>(
+ network_emulation_->time_controller()->GetClock());
+ video_quality_analyzer_ = video_quality_analyzer.get();
+ fixture_ = CreatePeerConnectionE2EQualityTestFixture(
+ testing::UnitTest::GetInstance()->current_test_info()->name(),
+ *network_emulation_->time_controller(),
/*audio_quality_analyzer=*/nullptr, std::move(video_quality_analyzer));
- fixture->ExecuteAt(TimeDelta::Seconds(1),
- [alice_network_behavior_ptr](TimeDelta) {
- BuiltInNetworkBehaviorConfig config;
- config.loss_percent = 5;
- alice_network_behavior_ptr->SetConfig(config);
- });
+ test::ScopedFieldTrials field_trials(
+ std::string(field_trial::GetFieldTrialString()) +
+ "WebRTC-UseStandardBytesStats/Enabled/");
+ }
- // Setup components. We need to provide rtc::NetworkManager compatible with
- // emulated network layer.
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ CreateNetwork() {
+ EmulatedNetworkNode* alice_node = network_emulation_->CreateEmulatedNode(
+ std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
+ EmulatedNetworkNode* bob_node = network_emulation_->CreateEmulatedNode(
+ std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
+
+ EmulatedEndpoint* alice_endpoint =
+ network_emulation_->CreateEndpoint(EmulatedEndpointConfig());
+ EmulatedEndpoint* bob_endpoint =
+ network_emulation_->CreateEndpoint(EmulatedEndpointConfig());
+
+ network_emulation_->CreateRoute(alice_endpoint, {alice_node}, bob_endpoint);
+ network_emulation_->CreateRoute(bob_endpoint, {bob_node}, alice_endpoint);
+
EmulatedNetworkManagerInterface* alice_network =
- network_emulation_manager->CreateEmulatedNetworkManagerInterface(
+ network_emulation_->CreateEmulatedNetworkManagerInterface(
{alice_endpoint});
EmulatedNetworkManagerInterface* bob_network =
- network_emulation_manager->CreateEmulatedNetworkManagerInterface(
+ network_emulation_->CreateEmulatedNetworkManagerInterface(
{bob_endpoint});
- fixture->AddPeer(alice_network->network_thread(),
- alice_network->network_manager(), alice_configurer);
- fixture->AddPeer(bob_network->network_thread(),
- bob_network->network_manager(), bob_configurer);
- fixture->AddQualityMetricsReporter(
- std::make_unique<NetworkQualityMetricsReporter>(alice_network,
- bob_network));
+ return std::make_pair(alice_network, bob_network);
+ }
- fixture->Run(run_params);
+ void AddPeer(EmulatedNetworkManagerInterface* network,
+ rtc::FunctionView<void(PeerConfigurer*)> configurer) {
+ fixture_->AddPeer(network->network_thread(), network->network_manager(),
+ configurer);
+ }
- EXPECT_GE(fixture->GetRealTestDuration(), run_params.run_duration);
- for (auto stream_key : video_analyzer_ptr->GetKnownVideoStreams()) {
+ void RunAndCheckEachVideoStreamReceivedFrames(const RunParams& run_params) {
+ fixture_->Run(run_params);
+
+ EXPECT_GE(fixture_->GetRealTestDuration(), run_params.run_duration);
+ for (auto stream_key : video_quality_analyzer_->GetKnownVideoStreams()) {
FrameCounters stream_conters =
- video_analyzer_ptr->GetPerStreamCounters().at(stream_key);
+ video_quality_analyzer_->GetPerStreamCounters().at(stream_key);
// On some devices the pipeline can be too slow, so we actually can't
// force real constraints here. Lets just check, that at least 1
// frame passed whole pipeline.
@@ -127,9 +111,18 @@
EXPECT_GE(stream_conters.rendered, 1) << stream_key.ToString();
}
}
-};
-} // namespace
+ NetworkEmulationManager* network_emulation() {
+ return network_emulation_.get();
+ }
+
+ PeerConnectionE2EQualityTestFixture* fixture() { return fixture_.get(); }
+
+ private:
+ std::unique_ptr<NetworkEmulationManager> network_emulation_;
+ DefaultVideoQualityAnalyzer* video_quality_analyzer_;
+ std::unique_ptr<PeerConnectionE2EQualityTestFixture> fixture_;
+};
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
@@ -138,46 +131,108 @@
#define MAYBE_Smoke Smoke
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "alice-video";
+ video.sync_group = "alice-media";
+ alice->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ audio.sampling_frequency_in_hz = 48000;
+ audio.sync_group = "alice-media";
+ alice->SetAudioConfig(std::move(audio));
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* charlie) {
+ charlie->SetName("charlie");
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "charlie-video";
+ video.temporal_layers_count = 2;
+ charlie->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "charlie-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
+ charlie->SetAudioConfig(std::move(audio));
+ });
+ fixture()->AddQualityMetricsReporter(
+ std::make_unique<NetworkQualityMetricsReporter>(network_links.first,
+ network_links.second));
RunParams run_params(TimeDelta::Seconds(2));
run_params.video_codecs = {
VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
run_params.use_flex_fec = true;
run_params.use_ulp_fec = true;
run_params.video_encoder_bitrate_multiplier = 1.1;
- test::ScopedFieldTrials field_trials(
- std::string(field_trial::GetFieldTrialString()) +
- "WebRTC-UseStandardBytesStats/Enabled/");
- RunTest(
- "smoke", run_params,
- [](PeerConfigurer* alice) {
- VideoConfig video(160, 120, 15);
- video.stream_label = "alice-video";
- video.sync_group = "alice-media";
- alice->AddVideoConfig(std::move(video));
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
- AudioConfig audio;
- audio.stream_label = "alice-audio";
- audio.mode = AudioConfig::Mode::kFile;
- audio.input_file_name =
- test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
- audio.sampling_frequency_in_hz = 48000;
- audio.sync_group = "alice-media";
- alice->SetAudioConfig(std::move(audio));
- },
- [](PeerConfigurer* charlie) {
- charlie->SetName("charlie");
- VideoConfig video(160, 120, 15);
- video.stream_label = "charlie-video";
- video.temporal_layers_count = 2;
- charlie->AddVideoConfig(std::move(video));
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_ChangeNetworkConditions DISABLED_ChangeNetworkConditions
+#else
+#define MAYBE_ChangeNetworkConditions ChangeNetworkConditions
+#endif
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) {
+ NetworkEmulationManager::SimulatedNetworkNode alice_node =
+ network_emulation()
+ ->NodeBuilder()
+ .config(BuiltInNetworkBehaviorConfig())
+ .Build();
+ NetworkEmulationManager::SimulatedNetworkNode bob_node =
+ network_emulation()
+ ->NodeBuilder()
+ .config(BuiltInNetworkBehaviorConfig())
+ .Build();
- AudioConfig audio;
- audio.stream_label = "charlie-audio";
- audio.mode = AudioConfig::Mode::kFile;
- audio.input_file_name =
- test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
- charlie->SetAudioConfig(std::move(audio));
- });
+ EmulatedEndpoint* alice_endpoint =
+ network_emulation()->CreateEndpoint(EmulatedEndpointConfig());
+ EmulatedEndpoint* bob_endpoint =
+ network_emulation()->CreateEndpoint(EmulatedEndpointConfig());
+
+ network_emulation()->CreateRoute(alice_endpoint, {alice_node.node},
+ bob_endpoint);
+ network_emulation()->CreateRoute(bob_endpoint, {bob_node.node},
+ alice_endpoint);
+
+ EmulatedNetworkManagerInterface* alice_network =
+ network_emulation()->CreateEmulatedNetworkManagerInterface(
+ {alice_endpoint});
+ EmulatedNetworkManagerInterface* bob_network =
+ network_emulation()->CreateEmulatedNetworkManagerInterface(
+ {bob_endpoint});
+
+ AddPeer(alice_network, [](PeerConfigurer* alice) {
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "alice-video";
+ video.sync_group = "alice-media";
+ alice->AddVideoConfig(std::move(video));
+ });
+ AddPeer(bob_network, [](PeerConfigurer* bob) {});
+ fixture()->AddQualityMetricsReporter(
+ std::make_unique<NetworkQualityMetricsReporter>(alice_network,
+ bob_network));
+
+ fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) {
+ BuiltInNetworkBehaviorConfig config;
+ config.loss_percent = 5;
+ alice_node.simulation->SetConfig(config);
+ });
+
+ RunParams run_params(TimeDelta::Seconds(2));
+ run_params.video_codecs = {
+ VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
+ run_params.use_flex_fec = true;
+ run_params.use_ulp_fec = true;
+ run_params.video_encoder_bitrate_multiplier = 1.1;
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
@@ -187,13 +242,10 @@
#define MAYBE_Screenshare Screenshare
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) {
- RunParams run_params(TimeDelta::Seconds(2));
- test::ScopedFieldTrials field_trials(
- std::string(field_trial::GetFieldTrialString()) +
- "WebRTC-UseStandardBytesStats/Enabled/");
- RunTest(
- "screenshare", run_params,
- [](PeerConfigurer* alice) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(
+ network_links.first, [](PeerConfigurer* alice) {
VideoConfig screenshare(320, 180, 30);
screenshare.stream_label = "alice-screenshare";
screenshare.content_hint = VideoTrackInterface::ContentHint::kText;
@@ -205,8 +257,9 @@
CreateScreenShareFrameGenerator(screenshare, screen_share_config);
alice->AddVideoConfig(std::move(screenshare),
std::move(screen_share_frame_generator));
- },
- [](PeerConfigurer* charlie) {});
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {});
+ RunAndCheckEachVideoStreamReceivedFrames(RunParams(TimeDelta::Seconds(2)));
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
@@ -216,27 +269,28 @@
#define MAYBE_Echo Echo
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ audio.sampling_frequency_in_hz = 48000;
+ alice->SetAudioConfig(std::move(audio));
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {
+ AudioConfig audio;
+ audio.stream_label = "bob-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
+ bob->SetAudioConfig(std::move(audio));
+ });
RunParams run_params(TimeDelta::Seconds(2));
run_params.echo_emulation_config = EchoEmulationConfig();
- RunTest(
- "smoke", run_params,
- [](PeerConfigurer* alice) {
- AudioConfig audio;
- audio.stream_label = "alice-audio";
- audio.mode = AudioConfig::Mode::kFile;
- audio.input_file_name =
- test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
- audio.sampling_frequency_in_hz = 48000;
- alice->SetAudioConfig(std::move(audio));
- },
- [](PeerConfigurer* bob) {
- AudioConfig audio;
- audio.stream_label = "bob-audio";
- audio.mode = AudioConfig::Mode::kFile;
- audio.input_file_name =
- test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
- bob->SetAudioConfig(std::move(audio));
- });
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
@@ -246,24 +300,25 @@
#define MAYBE_Simulcast Simulcast
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ VideoConfig simulcast(1280, 720, 15);
+ simulcast.stream_label = "alice-simulcast";
+ simulcast.simulcast_config = VideoSimulcastConfig(2, 0);
+ alice->AddVideoConfig(std::move(simulcast));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ alice->SetAudioConfig(std::move(audio));
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {});
RunParams run_params(TimeDelta::Seconds(2));
run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)};
- RunTest(
- "simulcast", run_params,
- [](PeerConfigurer* alice) {
- VideoConfig simulcast(1280, 720, 15);
- simulcast.stream_label = "alice-simulcast";
- simulcast.simulcast_config = VideoSimulcastConfig(2, 0);
- alice->AddVideoConfig(std::move(simulcast));
-
- AudioConfig audio;
- audio.stream_label = "alice-audio";
- audio.mode = AudioConfig::Mode::kFile;
- audio.input_file_name =
- test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
- alice->SetAudioConfig(std::move(audio));
- },
- [](PeerConfigurer* bob) {});
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
@@ -273,26 +328,27 @@
#define MAYBE_Svc Svc
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ VideoConfig simulcast(1280, 720, 15);
+ simulcast.stream_label = "alice-svc";
+ // Because we have network with packets loss we can analyze only the
+ // highest spatial layer in SVC mode.
+ simulcast.simulcast_config = VideoSimulcastConfig(2, 1);
+ alice->AddVideoConfig(std::move(simulcast));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ alice->SetAudioConfig(std::move(audio));
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {});
RunParams run_params(TimeDelta::Seconds(2));
run_params.video_codecs = {VideoCodecConfig(cricket::kVp9CodecName)};
- RunTest(
- "simulcast", run_params,
- [](PeerConfigurer* alice) {
- VideoConfig simulcast(1280, 720, 15);
- simulcast.stream_label = "alice-svc";
- // Because we have network with packets loss we can analyze only the
- // highest spatial layer in SVC mode.
- simulcast.simulcast_config = VideoSimulcastConfig(2, 1);
- alice->AddVideoConfig(std::move(simulcast));
-
- AudioConfig audio;
- audio.stream_label = "alice-audio";
- audio.mode = AudioConfig::Mode::kFile;
- audio.input_file_name =
- test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
- alice->SetAudioConfig(std::move(audio));
- },
- [](PeerConfigurer* bob) {});
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
@@ -302,33 +358,34 @@
#define MAYBE_HighBitrate HighBitrate
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ BitrateSettings bitrate_settings;
+ bitrate_settings.start_bitrate_bps = 3'000'000;
+ bitrate_settings.max_bitrate_bps = 3'000'000;
+ alice->SetBitrateSettings(bitrate_settings);
+ VideoConfig video(800, 600, 15);
+ video.stream_label = "alice-video";
+ video.min_encode_bitrate_bps = 500'000;
+ video.max_encode_bitrate_bps = 3'000'000;
+ alice->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ audio.sampling_frequency_in_hz = 48000;
+ alice->SetAudioConfig(std::move(audio));
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {});
RunParams run_params(TimeDelta::Seconds(2));
run_params.video_codecs = {
VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
-
- RunTest(
- "smoke", run_params,
- [](PeerConfigurer* alice) {
- BitrateSettings bitrate_settings;
- bitrate_settings.start_bitrate_bps = 3'000'000;
- bitrate_settings.max_bitrate_bps = 3'000'000;
- alice->SetBitrateSettings(bitrate_settings);
- VideoConfig video(800, 600, 15);
- video.stream_label = "alice-video";
- video.min_encode_bitrate_bps = 500'000;
- video.max_encode_bitrate_bps = 3'000'000;
- alice->AddVideoConfig(std::move(video));
-
- AudioConfig audio;
- audio.stream_label = "alice-audio";
- audio.mode = AudioConfig::Mode::kFile;
- audio.input_file_name =
- test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
- audio.sampling_frequency_in_hz = 48000;
- alice->SetAudioConfig(std::move(audio));
- },
- [](PeerConfigurer* bob) {});
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
}
+} // namespace
} // namespace webrtc_pc_e2e
} // namespace webrtc
