blob: b09b093c25b918e2d7ae6ac672681d3daa8fd0c6 [file] [log] [blame]
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <cstdint>
#include <memory>
#include "api/media_stream_interface.h"
#include "api/test/create_network_emulation_manager.h"
#include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h"
#include "call/simulated_network.h"
#include "system_wrappers/include/field_trial.h"
#include "test/field_trial.h"
#include "test/gtest.h"
#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
#include "test/pc/e2e/network_quality_metrics_reporter.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
namespace webrtc_pc_e2e {
namespace {
class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test {
public:
using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer;
using RunParams = PeerConnectionE2EQualityTestFixture::RunParams;
using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
using VideoCodecConfig =
PeerConnectionE2EQualityTestFixture::VideoCodecConfig;
using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig;
using ScreenShareConfig =
PeerConnectionE2EQualityTestFixture::ScreenShareConfig;
using ScrollingParams = PeerConnectionE2EQualityTestFixture::ScrollingParams;
using VideoSimulcastConfig =
PeerConnectionE2EQualityTestFixture::VideoSimulcastConfig;
using EchoEmulationConfig =
PeerConnectionE2EQualityTestFixture::EchoEmulationConfig;
void RunTest(const std::string& test_case_name,
const RunParams& run_params,
rtc::FunctionView<void(PeerConfigurer*)> alice_configurer,
rtc::FunctionView<void(PeerConfigurer*)> bob_configurer) {
// Setup emulated network
std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
CreateNetworkEmulationManager();
auto alice_network_behavior =
std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig());
SimulatedNetwork* alice_network_behavior_ptr = alice_network_behavior.get();
EmulatedNetworkNode* alice_node =
network_emulation_manager->CreateEmulatedNode(
std::move(alice_network_behavior));
EmulatedNetworkNode* bob_node =
network_emulation_manager->CreateEmulatedNode(
std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
auto* alice_endpoint =
network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
EmulatedEndpoint* bob_endpoint =
network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
network_emulation_manager->CreateRoute(alice_endpoint, {alice_node},
bob_endpoint);
network_emulation_manager->CreateRoute(bob_endpoint, {bob_node},
alice_endpoint);
// Create analyzers.
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer =
std::make_unique<DefaultVideoQualityAnalyzer>(
network_emulation_manager->time_controller()->GetClock());
// This is only done for the sake of smoke testing. In general there should
// be no need to explicitly pull data from analyzers after the run.
auto* video_analyzer_ptr =
static_cast<DefaultVideoQualityAnalyzer*>(video_quality_analyzer.get());
auto fixture = CreatePeerConnectionE2EQualityTestFixture(
test_case_name, *network_emulation_manager->time_controller(),
/*audio_quality_analyzer=*/nullptr, std::move(video_quality_analyzer));
fixture->ExecuteAt(TimeDelta::Seconds(1),
[alice_network_behavior_ptr](TimeDelta) {
BuiltInNetworkBehaviorConfig config;
config.loss_percent = 5;
alice_network_behavior_ptr->SetConfig(config);
});
// Setup components. We need to provide rtc::NetworkManager compatible with
// emulated network layer.
EmulatedNetworkManagerInterface* alice_network =
network_emulation_manager->CreateEmulatedNetworkManagerInterface(
{alice_endpoint});
EmulatedNetworkManagerInterface* bob_network =
network_emulation_manager->CreateEmulatedNetworkManagerInterface(
{bob_endpoint});
fixture->AddPeer(alice_network->network_thread(),
alice_network->network_manager(), alice_configurer);
fixture->AddPeer(bob_network->network_thread(),
bob_network->network_manager(), bob_configurer);
fixture->AddQualityMetricsReporter(
std::make_unique<NetworkQualityMetricsReporter>(alice_network,
bob_network));
fixture->Run(run_params);
EXPECT_GE(fixture->GetRealTestDuration(), run_params.run_duration);
for (auto stream_key : video_analyzer_ptr->GetKnownVideoStreams()) {
FrameCounters stream_conters =
video_analyzer_ptr->GetPerStreamCounters().at(stream_key);
// On some devices the pipeline can be too slow, so we actually can't
// force real constraints here. Lets just check, that at least 1
// frame passed whole pipeline.
int64_t expected_min_fps = run_params.run_duration.seconds() * 15;
EXPECT_GE(stream_conters.captured, expected_min_fps)
<< stream_key.ToString();
EXPECT_GE(stream_conters.pre_encoded, 1) << stream_key.ToString();
EXPECT_GE(stream_conters.encoded, 1) << stream_key.ToString();
EXPECT_GE(stream_conters.received, 1) << stream_key.ToString();
EXPECT_GE(stream_conters.decoded, 1) << stream_key.ToString();
EXPECT_GE(stream_conters.rendered, 1) << stream_key.ToString();
}
}
};
} // namespace
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
#define MAYBE_Smoke DISABLED_Smoke
#else
#define MAYBE_Smoke Smoke
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) {
RunParams run_params(TimeDelta::Seconds(2));
run_params.video_codecs = {
VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
run_params.use_flex_fec = true;
run_params.use_ulp_fec = true;
run_params.video_encoder_bitrate_multiplier = 1.1;
test::ScopedFieldTrials field_trials(
std::string(field_trial::GetFieldTrialString()) +
"WebRTC-UseStandardBytesStats/Enabled/");
RunTest(
"smoke", run_params,
[](PeerConfigurer* alice) {
VideoConfig video(160, 120, 15);
video.stream_label = "alice-video";
video.sync_group = "alice-media";
alice->AddVideoConfig(std::move(video));
AudioConfig audio;
audio.stream_label = "alice-audio";
audio.mode = AudioConfig::Mode::kFile;
audio.input_file_name =
test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
audio.sampling_frequency_in_hz = 48000;
audio.sync_group = "alice-media";
alice->SetAudioConfig(std::move(audio));
},
[](PeerConfigurer* charlie) {
charlie->SetName("charlie");
VideoConfig video(160, 120, 15);
video.stream_label = "charlie-video";
video.temporal_layers_count = 2;
charlie->AddVideoConfig(std::move(video));
AudioConfig audio;
audio.stream_label = "charlie-audio";
audio.mode = AudioConfig::Mode::kFile;
audio.input_file_name =
test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
charlie->SetAudioConfig(std::move(audio));
});
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
#define MAYBE_Screenshare DISABLED_Screenshare
#else
#define MAYBE_Screenshare Screenshare
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) {
RunParams run_params(TimeDelta::Seconds(2));
test::ScopedFieldTrials field_trials(
std::string(field_trial::GetFieldTrialString()) +
"WebRTC-UseStandardBytesStats/Enabled/");
RunTest(
"screenshare", run_params,
[](PeerConfigurer* alice) {
VideoConfig screenshare(320, 180, 30);
screenshare.stream_label = "alice-screenshare";
screenshare.content_hint = VideoTrackInterface::ContentHint::kText;
ScreenShareConfig screen_share_config =
ScreenShareConfig(TimeDelta::Seconds(2));
screen_share_config.scrolling_params = ScrollingParams(
TimeDelta::Millis(1800), kDefaultSlidesWidth, kDefaultSlidesHeight);
auto screen_share_frame_generator =
CreateScreenShareFrameGenerator(screenshare, screen_share_config);
alice->AddVideoConfig(std::move(screenshare),
std::move(screen_share_frame_generator));
},
[](PeerConfigurer* charlie) {});
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
#define MAYBE_Echo DISABLED_Echo
#else
#define MAYBE_Echo Echo
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) {
RunParams run_params(TimeDelta::Seconds(2));
run_params.echo_emulation_config = EchoEmulationConfig();
RunTest(
"smoke", run_params,
[](PeerConfigurer* alice) {
AudioConfig audio;
audio.stream_label = "alice-audio";
audio.mode = AudioConfig::Mode::kFile;
audio.input_file_name =
test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
audio.sampling_frequency_in_hz = 48000;
alice->SetAudioConfig(std::move(audio));
},
[](PeerConfigurer* bob) {
AudioConfig audio;
audio.stream_label = "bob-audio";
audio.mode = AudioConfig::Mode::kFile;
audio.input_file_name =
test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
bob->SetAudioConfig(std::move(audio));
});
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
#define MAYBE_Simulcast DISABLED_Simulcast
#else
#define MAYBE_Simulcast Simulcast
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) {
RunParams run_params(TimeDelta::Seconds(2));
run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)};
RunTest(
"simulcast", run_params,
[](PeerConfigurer* alice) {
VideoConfig simulcast(1280, 720, 15);
simulcast.stream_label = "alice-simulcast";
simulcast.simulcast_config = VideoSimulcastConfig(2, 0);
alice->AddVideoConfig(std::move(simulcast));
AudioConfig audio;
audio.stream_label = "alice-audio";
audio.mode = AudioConfig::Mode::kFile;
audio.input_file_name =
test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
alice->SetAudioConfig(std::move(audio));
},
[](PeerConfigurer* bob) {});
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
#define MAYBE_Svc DISABLED_Svc
#else
#define MAYBE_Svc Svc
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) {
RunParams run_params(TimeDelta::Seconds(2));
run_params.video_codecs = {VideoCodecConfig(cricket::kVp9CodecName)};
RunTest(
"simulcast", run_params,
[](PeerConfigurer* alice) {
VideoConfig simulcast(1280, 720, 15);
simulcast.stream_label = "alice-svc";
// Because we have network with packets loss we can analyze only the
// highest spatial layer in SVC mode.
simulcast.simulcast_config = VideoSimulcastConfig(2, 1);
alice->AddVideoConfig(std::move(simulcast));
AudioConfig audio;
audio.stream_label = "alice-audio";
audio.mode = AudioConfig::Mode::kFile;
audio.input_file_name =
test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
alice->SetAudioConfig(std::move(audio));
},
[](PeerConfigurer* bob) {});
}
// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
#define MAYBE_HighBitrate DISABLED_HighBitrate
#else
#define MAYBE_HighBitrate HighBitrate
#endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) {
RunParams run_params(TimeDelta::Seconds(2));
run_params.video_codecs = {
VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
RunTest(
"smoke", run_params,
[](PeerConfigurer* alice) {
BitrateSettings bitrate_settings;
bitrate_settings.start_bitrate_bps = 3'000'000;
bitrate_settings.max_bitrate_bps = 3'000'000;
alice->SetBitrateSettings(bitrate_settings);
VideoConfig video(800, 600, 15);
video.stream_label = "alice-video";
video.min_encode_bitrate_bps = 500'000;
video.max_encode_bitrate_bps = 3'000'000;
alice->AddVideoConfig(std::move(video));
AudioConfig audio;
audio.stream_label = "alice-audio";
audio.mode = AudioConfig::Mode::kFile;
audio.input_file_name =
test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
audio.sampling_frequency_in_hz = 48000;
alice->SetAudioConfig(std::move(audio));
},
[](PeerConfigurer* bob) {});
}
} // namespace webrtc_pc_e2e
} // namespace webrtc