Enable audio in video_quality_test.
Allows enabling audio for RunWithAnalyzer method, and prints out audio jitterbuffer performance stats. Also fixes for RunWithRenderer when enabling audio (seg-faulted).
Bug: b/112299470
Change-Id: Ic7c0de1c455891f38cca317001c6c216e82f6ec3
Reviewed-on: https://webrtc-review.googlesource.com/92800
Commit-Queue: Christoffer Rodbro <crodbro@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24208}diff --git a/video/video_analyzer.cc b/video/video_analyzer.cc
index 480e3ea..66c4db6 100644
--- a/video/video_analyzer.cc
+++ b/video/video_analyzer.cc
@@ -60,6 +60,7 @@
call_(nullptr),
send_stream_(nullptr),
receive_stream_(nullptr),
+ audio_receive_stream_(nullptr),
captured_frame_forwarder_(this, clock),
test_label_(test_label),
graph_data_output_file_(graph_data_output_file),
@@ -164,6 +165,12 @@
receive_stream_ = stream;
}
+void VideoAnalyzer::SetAudioReceiveStream(AudioReceiveStream* recv_stream) {
+ rtc::CritScope lock(&crit_);
+ RTC_CHECK(!audio_receive_stream_);
+ audio_receive_stream_ = recv_stream;
+}
+
rtc::VideoSinkInterface<VideoFrame>* VideoAnalyzer::InputInterface() {
return &captured_frame_forwarder_;
}
@@ -460,6 +467,14 @@
decode_time_max_ms_.AddSample(receive_stats.max_decode_ms);
}
+ if (audio_receive_stream_ != nullptr) {
+ AudioReceiveStream::Stats receive_stats =
+ audio_receive_stream_->GetStats();
+ audio_expand_rate_.AddSample(receive_stats.expand_rate);
+ audio_accelerate_rate_.AddSample(receive_stats.accelerate_rate);
+ audio_jitter_buffer_ms_.AddSample(receive_stats.jitter_buffer_ms);
+ }
+
memory_usage_.AddSample(rtc::GetProcessResidentSizeBytes());
}
}
@@ -594,6 +609,12 @@
frame_writer.WriteFrame(worst_frame_->frame, 100 /*best quality*/));
}
+ if (audio_receive_stream_ != nullptr) {
+ PrintResult("audio_expand_rate", audio_expand_rate_, "");
+ PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "");
+ PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, " ms");
+ }
+
// Disable quality check for quick test, as quality checks may fail
// because too few samples were collected.
if (!is_quick_test_enabled_) {
diff --git a/video/video_analyzer.h b/video/video_analyzer.h
index 1b5a87d..b4d31db 100644
--- a/video/video_analyzer.h
+++ b/video/video_analyzer.h
@@ -50,6 +50,8 @@
void SetCall(Call* call);
void SetSendStream(VideoSendStream* stream);
void SetReceiveStream(VideoReceiveStream* stream);
+ void SetAudioReceiveStream(AudioReceiveStream* recv_stream);
+
rtc::VideoSinkInterface<VideoFrame>* InputInterface();
rtc::VideoSourceInterface<VideoFrame>* OutputInterface();
@@ -208,6 +210,7 @@
Call* call_;
VideoSendStream* send_stream_;
VideoReceiveStream* receive_stream_;
+ AudioReceiveStream* audio_receive_stream_;
CapturedFrameForwarder captured_frame_forwarder_;
const std::string test_label_;
FILE* const graph_data_output_file_;
@@ -239,6 +242,9 @@
test::Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_);
test::Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
+ test::Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_);
+ test::Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_);
+ test::Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_);
// Rendered frame with worst PSNR is saved for further analysis.
absl::optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_);
diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc
index cad86ae..071fe5f 100644
--- a/video/video_quality_test.cc
+++ b/video/video_quality_test.cc
@@ -757,6 +757,24 @@
}
}
+void VideoQualityTest::StartThumbnails() {
+ for (VideoSendStream* send_stream : thumbnail_send_streams_)
+ send_stream->Start();
+ for (VideoReceiveStream* receive_stream : thumbnail_receive_streams_)
+ receive_stream->Start();
+ for (std::unique_ptr<test::VideoCapturer>& capturer : thumbnail_capturers_)
+ capturer->Start();
+}
+
+void VideoQualityTest::StopThumbnails() {
+ for (std::unique_ptr<test::VideoCapturer>& capturer : thumbnail_capturers_)
+ capturer->Stop();
+ for (VideoReceiveStream* receive_stream : thumbnail_receive_streams_)
+ receive_stream->Stop();
+ for (VideoSendStream* send_stream : thumbnail_send_streams_)
+ send_stream->Stop();
+}
+
std::unique_ptr<test::LayerFilteringTransport>
VideoQualityTest::CreateSendTransport() {
return absl::make_unique<test::LayerFilteringTransport>(
@@ -781,8 +799,6 @@
std::unique_ptr<VideoAnalyzer> analyzer;
params_ = params;
-
- RTC_CHECK(!params_.audio.enabled);
// TODO(ivica): Merge with RunWithRenderer and use a flag / argument to
// differentiate between the analyzer and the renderer case.
CheckParams();
@@ -824,6 +840,10 @@
task_queue_.SendTask([this, &send_call_config, &recv_call_config,
&send_transport, &recv_transport]() {
+ if (params_.audio.enabled) {
+ InitializeAudioDevice(&send_call_config, &recv_call_config);
+ }
+
CreateCalls(send_call_config, recv_call_config);
send_transport = CreateSendTransport();
recv_transport = CreateReceiveTransport();
@@ -886,39 +906,24 @@
StartEncodedFrameLogs(GetVideoSendStream());
StartEncodedFrameLogs(
video_receive_streams_[params_.ss[0].selected_stream]);
- StartVideoStreams();
- for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
- thumbnail_send_stream->Start();
- for (VideoReceiveStream* thumbnail_receive_stream :
- thumbnail_receive_streams_)
- thumbnail_receive_stream->Start();
- analyzer->StartMeasuringCpuProcessTime();
- StartVideoCapture();
- for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
- thumbnail_capturers_) {
- video_caputurer->Start();
+ if (params_.audio.enabled) {
+ SetupAudio(send_transport.get());
}
+ StartThumbnails();
+ Start();
+
+ if (params_.audio.enabled) {
+ analyzer->SetAudioReceiveStream(audio_receive_streams_[0]);
+ }
+ analyzer->StartMeasuringCpuProcessTime();
});
analyzer->Wait();
task_queue_.SendTask([&]() {
- for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
- thumbnail_capturers_)
- video_caputurer->Stop();
- for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) {
- video_capturers_[video_idx]->Stop();
- }
- for (VideoReceiveStream* thumbnail_receive_stream :
- thumbnail_receive_streams_)
- thumbnail_receive_stream->Stop();
- for (VideoReceiveStream* receive_stream : video_receive_streams_)
- receive_stream->Stop();
- for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
- thumbnail_send_stream->Stop();
- for (VideoSendStream* video_send_stream : video_send_streams_)
- video_send_stream->Stop();
+ StopThumbnails();
+ Stop();
DestroyStreams();
DestroyThumbnailStreams();
@@ -934,6 +939,24 @@
});
}
+void VideoQualityTest::InitializeAudioDevice(Call::Config* send_call_config,
+ Call::Config* recv_call_config) {
+ rtc::scoped_refptr<TestAudioDeviceModule> fake_audio_device =
+ TestAudioDeviceModule::CreateTestAudioDeviceModule(
+ TestAudioDeviceModule::CreatePulsedNoiseCapturer(32000, 48000),
+ TestAudioDeviceModule::CreateDiscardRenderer(48000), 1.f);
+
+ AudioState::Config audio_state_config;
+ audio_state_config.audio_mixer = AudioMixerImpl::Create();
+ audio_state_config.audio_processing = AudioProcessingBuilder().Create();
+ audio_state_config.audio_device_module = fake_audio_device;
+ send_call_config->audio_state = AudioState::Create(audio_state_config);
+ RTC_CHECK(fake_audio_device->RegisterAudioCallback(
+ send_call_config->audio_state->audio_transport()) == 0);
+ recv_call_config->audio_state = AudioState::Create(audio_state_config);
+ fake_audio_device->Init();
+}
+
void VideoQualityTest::SetupAudio(Transport* transport) {
AudioSendStream::Config audio_send_config(transport);
audio_send_config.rtp.ssrc = kAudioSendSsrc;
@@ -959,7 +982,7 @@
audio_send_config.encoder_factory = audio_encoder_factory_;
SetAudioConfig(audio_send_config);
- const char* sync_group = nullptr;
+ std::string sync_group;
if (params_.video[0].enabled && params_.audio.sync_video)
sync_group = kSyncGroup;
@@ -984,22 +1007,13 @@
Call::Config call_config(&null_event_log);
call_config.bitrate_config = params_.call.call_bitrate_config;
- rtc::scoped_refptr<TestAudioDeviceModule> fake_audio_device =
- TestAudioDeviceModule::CreateTestAudioDeviceModule(
- TestAudioDeviceModule::CreatePulsedNoiseCapturer(32000, 48000),
- TestAudioDeviceModule::CreateDiscardRenderer(48000), 1.f);
+ Call::Config recv_call_config(&null_event_log);
if (params_.audio.enabled) {
- AudioState::Config audio_state_config;
- audio_state_config.audio_mixer = AudioMixerImpl::Create();
- audio_state_config.audio_processing = AudioProcessingBuilder().Create();
- audio_state_config.audio_device_module = fake_audio_device;
- call_config.audio_state = AudioState::Create(audio_state_config);
- fake_audio_device->RegisterAudioCallback(
- call_config.audio_state->audio_transport());
+ InitializeAudioDevice(&call_config, &recv_call_config);
}
- CreateCalls(call_config, call_config);
+ CreateCalls(call_config, recv_call_config);
// TODO(minyue): consider if this is a good transport even for audio only
// calls.
diff --git a/video/video_quality_test.h b/video/video_quality_test.h
index 67facfd..f00c748 100644
--- a/video/video_quality_test.h
+++ b/video/video_quality_test.h
@@ -73,7 +73,11 @@
const SdpVideoFormat& format);
void SetupVideo(Transport* send_transport, Transport* recv_transport);
void SetupThumbnails(Transport* send_transport, Transport* recv_transport);
+ void StartThumbnails();
+ void StopThumbnails();
void DestroyThumbnailStreams();
+ void InitializeAudioDevice(Call::Config* send_call_config,
+ Call::Config* recv_call_config);
void SetupAudio(Transport* transport);
void StartEncodedFrameLogs(VideoSendStream* stream);
