blob: 571547e2bd52e1e45155366d03d42091a608233d [file] [log] [blame]
ivica5d6a06c2015-09-17 12:30:241/*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
Mirko Bonadei92ea95e2017-09-15 04:47:3110#include "video/video_quality_test.h"
perkj9fdbda62016-09-15 16:19:2011
perkja49cbd32016-09-16 14:53:4112#include <stdio.h>
Jonas Olssona4d87372019-07-05 17:08:3313
Tommi9e46cf52020-05-04 14:43:0514#if defined(WEBRTC_WIN)
15#include <conio.h>
16#endif
17
ivica5d6a06c2015-09-17 12:30:2418#include <algorithm>
19#include <deque>
20#include <map>
Mirko Bonadei317a1f02019-09-17 15:06:1821#include <memory>
mflodmand1590b22015-12-09 15:07:5922#include <string>
ivica5d6a06c2015-09-17 12:30:2423#include <vector>
24
Florent Castellif4673f92024-04-19 19:46:0325#include "api/audio/audio_device.h"
Elad Alon8f01c4e2019-06-28 13:19:4326#include "api/fec_controller_override.h"
Niels Möllerd8b9ed72019-05-08 11:53:5127#include "api/rtc_event_log_output_file.h"
Danil Chapovalov1c41be62019-04-01 07:16:1228#include "api/task_queue/default_task_queue_factory.h"
Danil Chapovalov44db4362019-09-30 02:16:2829#include "api/task_queue/task_queue_base.h"
Artem Titov33f9d2b2019-12-05 14:59:0030#include "api/test/create_frame_generator.h"
Jiawei Ouc2ebe212018-11-08 18:02:5631#include "api/video/builtin_video_bitrate_allocator_factory.h"
Elad Alon370f93a2019-06-11 12:57:5732#include "api/video_codecs/video_encoder.h"
Artem Titovdd2eebe2018-08-20 11:27:4533#include "call/fake_network_pipe.h"
Danil Chapovalova54ba4c2020-05-11 10:36:1334#include "media/base/media_constants.h"
henrika255750b2018-08-27 14:13:3735#include "media/engine/adm_helpers.h"
Emircan Uysaler7c03bdc2019-01-16 20:07:5636#include "media/engine/fake_video_codec_factory.h"
Steve Anton10542f22019-01-11 17:11:0037#include "media/engine/internal_encoder_factory.h"
Henrik Boströmf6eae952023-03-15 10:29:3038#include "media/engine/simulcast_encoder_adapter.h"
Steve Anton10542f22019-01-11 17:11:0039#include "media/engine/webrtc_video_engine.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3140#include "modules/audio_mixer/audio_mixer_impl.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3141#include "modules/video_coding/codecs/h264/include/h264.h"
Niels Möller4db138e2018-04-19 07:04:1342#include "modules/video_coding/codecs/vp8/include/vp8.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3143#include "modules/video_coding/codecs/vp9/include/vp9.h"
Niels Möller1beef1a2018-09-03 12:21:4444#include "modules/video_coding/utility/ivf_file_writer.h"
45#include "rtc_base/strings/string_builder.h"
Danil Chapovalov82a3f0a2019-10-21 07:24:2746#include "rtc_base/task_queue_for_test.h"
Per K569849e2024-04-29 08:37:3747#include "test/network/simulated_network.h"
Kári Tristan Helgasonede7cb22019-03-06 09:34:0948#include "test/platform_video_capturer.h"
Danil Chapovalovf2443a72023-10-09 14:39:4049#include "test/test_flags.h"
Steve Anton10542f22019-01-11 17:11:0050#include "test/testsupport/file_utils.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3151#include "test/video_renderer.h"
Erik Språng96965ae2018-10-23 13:42:3752#include "video/frame_dumping_decoder.h"
henrika255750b2018-08-27 14:13:3753#ifdef WEBRTC_WIN
54#include "modules/audio_device/include/audio_device_factory.h"
55#endif
Artem Titov8a9f3a82023-04-25 07:56:4956#include "test/video_test_constants.h"
Jonas Oreland1262eb52022-09-27 14:53:0457#include "video/config/encoder_stream_factory.h"
ilnikee42d192017-08-22 14:16:2058
Niels Möller1beef1a2018-09-03 12:21:4459namespace webrtc {
60
minyue73208662016-08-18 13:28:5561namespace {
Elad Alond8d32482019-02-18 22:45:5762enum : int { // The first valid value is 1.
63 kAbsSendTimeExtensionId = 1,
Elad Alonccb9b752019-02-19 12:01:3164 kGenericFrameDescriptorExtensionId00,
65 kGenericFrameDescriptorExtensionId01,
Elad Alond8d32482019-02-18 22:45:5766 kTransportSequenceNumberExtensionId,
67 kVideoContentTypeExtensionId,
68 kVideoTimingExtensionId,
69};
70
minyue73208662016-08-18 13:28:5571constexpr char kSyncGroup[] = "av_sync";
minyue10cbb462016-11-07 17:29:2272constexpr int kOpusMinBitrateBps = 6000;
73constexpr int kOpusBitrateFbBps = 32000;
ilnik9ae0d762017-02-15 08:53:1274constexpr int kFramesSentInQuickTest = 1;
ilnika014cc52017-03-07 12:21:0475constexpr uint32_t kThumbnailSendSsrcStart = 0xE0000;
76constexpr uint32_t kThumbnailRtxSsrcStart = 0xF0000;
minyue73208662016-08-18 13:28:5577
Elad Alon370f93a2019-06-11 12:57:5778const VideoEncoder::Capabilities kCapabilities(false);
79
Erik Språng616b2332019-02-11 13:16:2880std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec,
81 size_t spatial_idx) {
82 uint32_t min_bitrate = codec.minBitrate;
83 uint32_t max_bitrate = codec.maxBitrate;
84 if (spatial_idx < codec.numberOfSimulcastStreams) {
85 min_bitrate =
86 std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate);
87 max_bitrate =
88 std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate);
89 }
90 if (codec.codecType == VideoCodecType::kVideoCodecVP9 &&
91 spatial_idx < codec.VP9().numberOfSpatialLayers) {
92 min_bitrate =
93 std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate);
94 max_bitrate =
95 std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate);
96 }
97 max_bitrate = std::max(max_bitrate, min_bitrate);
98 return {min_bitrate * 1000, max_bitrate * 1000};
99}
100
perkjfa10b552016-10-03 06:45:26101class VideoStreamFactory
Niels Möller1beef1a2018-09-03 12:21:44102 : public VideoEncoderConfig::VideoStreamFactoryInterface {
perkjfa10b552016-10-03 06:45:26103 public:
Niels Möller1beef1a2018-09-03 12:21:44104 explicit VideoStreamFactory(const std::vector<VideoStream>& streams)
perkjfa10b552016-10-03 06:45:26105 : streams_(streams) {}
106
107 private:
Niels Möller1beef1a2018-09-03 12:21:44108 std::vector<VideoStream> CreateEncoderStreams(
Danil Chapovalov44ab2002024-04-17 10:11:29109 const FieldTrialsView& /*field_trials*/,
Jonas Oreland80c87d72022-09-29 13:01:09110 int frame_width,
111 int frame_height,
Niels Möller1beef1a2018-09-03 12:21:44112 const VideoEncoderConfig& encoder_config) override {
mflodmand79f97b2016-12-15 15:24:33113 // The highest layer must match the incoming resolution.
Niels Möller1beef1a2018-09-03 12:21:44114 std::vector<VideoStream> streams = streams_;
Jonas Oreland80c87d72022-09-29 13:01:09115 streams[streams_.size() - 1].height = frame_height;
116 streams[streams_.size() - 1].width = frame_width;
Seth Hampson24722b32017-12-22 17:36:42117
118 streams[0].bitrate_priority = encoder_config.bitrate_priority;
mflodmand79f97b2016-12-15 15:24:33119 return streams;
perkjfa10b552016-10-03 06:45:26120 }
121
Niels Möller1beef1a2018-09-03 12:21:44122 std::vector<VideoStream> streams_;
perkjfa10b552016-10-03 06:45:26123};
ivica5d6a06c2015-09-17 12:30:24124
Niels Möller88be9722018-10-10 08:58:52125// This wrapper provides two features needed by the video quality tests:
126// 1. Invoke VideoAnalyzer callbacks before and after encoding each frame.
127// 2. Write the encoded frames to file, one file per simulcast layer.
128class QualityTestVideoEncoder : public VideoEncoder,
129 private EncodedImageCallback {
Niels Möller1beef1a2018-09-03 12:21:44130 public:
Niels Möller88be9722018-10-10 08:58:52131 QualityTestVideoEncoder(std::unique_ptr<VideoEncoder> encoder,
132 VideoAnalyzer* analyzer,
Erik Språng616b2332019-02-11 13:16:28133 std::vector<FileWrapper> files,
134 double overshoot_factor)
135 : encoder_(std::move(encoder)),
136 overshoot_factor_(overshoot_factor),
137 analyzer_(analyzer) {
Niels Möllerb7edf692019-02-08 15:40:53138 for (FileWrapper& file : files) {
Niels Möller1beef1a2018-09-03 12:21:44139 writers_.push_back(
Niels Möllerb7edf692019-02-08 15:40:53140 IvfFileWriter::Wrap(std::move(file), /* byte_limit= */ 100000000));
Niels Möller1beef1a2018-09-03 12:21:44141 }
142 }
Elad Alonfe4f6942019-05-07 21:13:42143
Niels Möller1beef1a2018-09-03 12:21:44144 // Implement VideoEncoder
Elad Alon8f01c4e2019-06-28 13:19:43145 void SetFecControllerOverride(
146 FecControllerOverride* fec_controller_override) {
147 // Ignored.
148 }
149
Niels Möller1beef1a2018-09-03 12:21:44150 int32_t InitEncode(const VideoCodec* codec_settings,
Elad Alon370f93a2019-06-11 12:57:57151 const Settings& settings) override {
Erik Språng616b2332019-02-11 13:16:28152 codec_settings_ = *codec_settings;
Elad Alon370f93a2019-06-11 12:57:57153 return encoder_->InitEncode(codec_settings, settings);
Niels Möller1beef1a2018-09-03 12:21:44154 }
Elad Alonfe4f6942019-05-07 21:13:42155
Niels Möller1beef1a2018-09-03 12:21:44156 int32_t RegisterEncodeCompleteCallback(
157 EncodedImageCallback* callback) override {
158 callback_ = callback;
159 return encoder_->RegisterEncodeCompleteCallback(this);
160 }
Elad Alonfe4f6942019-05-07 21:13:42161
Niels Möller1beef1a2018-09-03 12:21:44162 int32_t Release() override { return encoder_->Release(); }
Elad Alonfe4f6942019-05-07 21:13:42163
Niels Möller1beef1a2018-09-03 12:21:44164 int32_t Encode(const VideoFrame& frame,
Niels Möller87e2d782019-03-07 09:18:23165 const std::vector<VideoFrameType>* frame_types) {
Niels Möller88be9722018-10-10 08:58:52166 if (analyzer_) {
167 analyzer_->PreEncodeOnFrame(frame);
168 }
Niels Möllerc8d2e732019-03-06 11:00:33169 return encoder_->Encode(frame, frame_types);
Niels Möller1beef1a2018-09-03 12:21:44170 }
Elad Alonfe4f6942019-05-07 21:13:42171
Erik Språng16cb8f52019-04-12 11:59:09172 void SetRates(const RateControlParameters& parameters) override {
Erik Språng616b2332019-02-11 13:16:28173 RTC_DCHECK_GT(overshoot_factor_, 0.0);
174 if (overshoot_factor_ == 1.0) {
Erik Språng16cb8f52019-04-12 11:59:09175 encoder_->SetRates(parameters);
176 return;
Erik Språng616b2332019-02-11 13:16:28177 }
178
179 // Simulating encoder overshooting target bitrate, by configuring actual
180 // encoder too high. Take care not to adjust past limits of config,
181 // otherwise encoders may crash on DCHECK.
182 VideoBitrateAllocation overshot_allocation;
183 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
184 const uint32_t spatial_layer_bitrate_bps =
Erik Språng16cb8f52019-04-12 11:59:09185 parameters.bitrate.GetSpatialLayerSum(si);
Erik Språng616b2332019-02-11 13:16:28186 if (spatial_layer_bitrate_bps == 0) {
187 continue;
188 }
189
190 uint32_t min_bitrate_bps;
191 uint32_t max_bitrate_bps;
192 std::tie(min_bitrate_bps, max_bitrate_bps) =
193 GetMinMaxBitratesBps(codec_settings_, si);
194 double overshoot_factor = overshoot_factor_;
195 const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>(
196 overshoot_factor * spatial_layer_bitrate_bps);
197 if (corrected_bitrate < min_bitrate_bps) {
198 overshoot_factor = min_bitrate_bps / spatial_layer_bitrate_bps;
199 } else if (corrected_bitrate > max_bitrate_bps) {
200 overshoot_factor = max_bitrate_bps / spatial_layer_bitrate_bps;
201 }
202
203 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 11:59:09204 if (parameters.bitrate.HasBitrate(si, ti)) {
Erik Språng616b2332019-02-11 13:16:28205 overshot_allocation.SetBitrate(
206 si, ti,
Erik Språng16cb8f52019-04-12 11:59:09207 rtc::checked_cast<uint32_t>(
208 overshoot_factor * parameters.bitrate.GetBitrate(si, ti)));
Erik Språng616b2332019-02-11 13:16:28209 }
210 }
211 }
212
Erik Språng16cb8f52019-04-12 11:59:09213 return encoder_->SetRates(
214 RateControlParameters(overshot_allocation, parameters.framerate_fps,
215 parameters.bandwidth_allocation));
Niels Möller1beef1a2018-09-03 12:21:44216 }
Elad Alonfe4f6942019-05-07 21:13:42217
218 void OnPacketLossRateUpdate(float packet_loss_rate) override {
219 encoder_->OnPacketLossRateUpdate(packet_loss_rate);
220 }
221
222 void OnRttUpdate(int64_t rtt_ms) override { encoder_->OnRttUpdate(rtt_ms); }
223
224 void OnLossNotification(const LossNotification& loss_notification) override {
225 encoder_->OnLossNotification(loss_notification);
226 }
227
Erik Språng189013b2018-11-01 14:03:57228 EncoderInfo GetEncoderInfo() const override {
Erik Språng616b2332019-02-11 13:16:28229 EncoderInfo info = encoder_->GetEncoderInfo();
230 if (overshoot_factor_ != 1.0) {
231 // We're simulating bad encoder, don't forward trusted setting
232 // from eg libvpx.
233 info.has_trusted_rate_controller = false;
234 }
235 return info;
Niels Möller1beef1a2018-09-03 12:21:44236 }
237
238 private:
239 // Implement EncodedImageCallback
240 Result OnEncodedImage(const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 15:30:36241 const CodecSpecificInfo* codec_specific_info) override {
Niels Möller1beef1a2018-09-03 12:21:44242 if (codec_specific_info) {
Henrik Boströmc5a4c932023-02-21 14:53:43243 int simulcast_index = encoded_image.SimulcastIndex().value_or(0);
Niels Möller1beef1a2018-09-03 12:21:44244 RTC_DCHECK_GE(simulcast_index, 0);
Niels Möller88be9722018-10-10 08:58:52245 if (analyzer_) {
246 analyzer_->PostEncodeOnFrame(simulcast_index,
Danil Chapovalov9c584832023-09-18 13:48:49247 encoded_image.RtpTimestamp());
Niels Möller88be9722018-10-10 08:58:52248 }
Niels Möller1beef1a2018-09-03 12:21:44249 if (static_cast<size_t>(simulcast_index) < writers_.size()) {
250 writers_[simulcast_index]->WriteFrame(encoded_image,
251 codec_specific_info->codecType);
252 }
253 }
254
Danil Chapovalov2549f172020-08-12 15:30:36255 return callback_->OnEncodedImage(encoded_image, codec_specific_info);
Niels Möller1beef1a2018-09-03 12:21:44256 }
257
258 void OnDroppedFrame(DropReason reason) override {
259 callback_->OnDroppedFrame(reason);
260 }
261
Erik Språng616b2332019-02-11 13:16:28262 const std::unique_ptr<VideoEncoder> encoder_;
263 const double overshoot_factor_;
Niels Möller88be9722018-10-10 08:58:52264 VideoAnalyzer* const analyzer_;
Niels Möller1beef1a2018-09-03 12:21:44265 std::vector<std::unique_ptr<IvfFileWriter>> writers_;
Erik Språng616b2332019-02-11 13:16:28266 EncodedImageCallback* callback_ = nullptr;
267 VideoCodec codec_settings_;
Niels Möller1beef1a2018-09-03 12:21:44268};
269
Tommi9e46cf52020-05-04 14:43:05270#if defined(WEBRTC_WIN) && !defined(WINUWP)
271void PressEnterToContinue(TaskQueueBase* task_queue) {
272 puts(">> Press ENTER to continue...");
273
274 while (!_kbhit() || _getch() != '\r') {
275 // Drive the message loop for the thread running the task_queue
Danil Chapovalove519f382022-08-11 10:26:09276 SendTask(task_queue, [&]() {
Tommi9e46cf52020-05-04 14:43:05277 MSG msg;
278 if (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) {
279 TranslateMessage(&msg);
280 DispatchMessage(&msg);
281 }
282 });
283 }
284}
285#else
286void PressEnterToContinue(TaskQueueBase* /*task_queue*/) {
287 puts(">> Press ENTER to continue...");
288 while (getc(stdin) != '\n' && !feof(stdin))
289 ; // NOLINT
290}
291#endif
292
Niels Möller1beef1a2018-09-03 12:21:44293} // namespace
ivica5d6a06c2015-09-17 12:30:24294
Niels Möllercbcbc222018-09-28 07:07:24295std::unique_ptr<VideoDecoder> VideoQualityTest::CreateVideoDecoder(
Danil Chapovalov61b1f532024-01-29 09:36:53296 const Environment& env,
Niels Möllercbcbc222018-09-28 07:07:24297 const SdpVideoFormat& format) {
298 std::unique_ptr<VideoDecoder> decoder;
Markus Handell97df9322024-02-23 11:12:54299 if (format.name == "FakeCodec") {
Emircan Uysaler7c03bdc2019-01-16 20:07:56300 decoder = webrtc::FakeVideoDecoderFactory::CreateVideoDecoder();
Niels Möllercbcbc222018-09-28 07:07:24301 } else {
Danil Chapovalov61b1f532024-01-29 09:36:53302 decoder = decoder_factory_->Create(env, format);
Niels Möllercbcbc222018-09-28 07:07:24303 }
Niels Möller5ca29122018-10-02 07:55:01304 if (!params_.logging.encoded_frame_base_path.empty()) {
305 rtc::StringBuilder str;
306 str << receive_logs_++;
307 std::string path =
308 params_.logging.encoded_frame_base_path + "." + str.str() + ".recv.ivf";
Markus Handell1c2f6372019-08-20 18:21:37309 decoder = CreateFrameDumpingDecoderWrapper(
Niels Möllerb7edf692019-02-08 15:40:53310 std::move(decoder), FileWrapper::OpenWriteOnly(path));
Niels Möller5ca29122018-10-02 07:55:01311 }
Niels Möllercbcbc222018-09-28 07:07:24312 return decoder;
313}
314
Sebastian Janssone6d7c3e2018-07-11 13:00:41315std::unique_ptr<VideoEncoder> VideoQualityTest::CreateVideoEncoder(
Danil Chapovalov329f0ea2024-03-08 13:30:54316 const Environment& env,
Niels Möller88be9722018-10-10 08:58:52317 const SdpVideoFormat& format,
318 VideoAnalyzer* analyzer) {
Niels Möller1beef1a2018-09-03 12:21:44319 std::unique_ptr<VideoEncoder> encoder;
Sergey Silkinc8b857f2024-06-22 11:35:19320 if (format.name == "FakeCodec") {
Danil Chapovalov6f1d4e72024-03-21 17:28:16321 encoder = FakeVideoEncoderFactory().Create(env, format);
Niels Möller1beef1a2018-09-03 12:21:44322 } else {
Sergey Silkinc8b857f2024-06-22 11:35:19323 encoder = std::make_unique<SimulcastEncoderAdapter>(
324 env, encoder_factory_.get(), nullptr, format);
Niels Möller4db138e2018-04-19 07:04:13325 }
Erik Språng616b2332019-02-11 13:16:28326
327 std::vector<FileWrapper> encoded_frame_dump_files;
Niels Möller1beef1a2018-09-03 12:21:44328 if (!params_.logging.encoded_frame_base_path.empty()) {
329 char ss_buf[100];
330 rtc::SimpleStringBuilder sb(ss_buf);
331 sb << send_logs_++;
332 std::string prefix =
333 params_.logging.encoded_frame_base_path + "." + sb.str() + ".send.";
Erik Språng616b2332019-02-11 13:16:28334 encoded_frame_dump_files.push_back(
335 FileWrapper::OpenWriteOnly(prefix + "1.ivf"));
336 encoded_frame_dump_files.push_back(
337 FileWrapper::OpenWriteOnly(prefix + "2.ivf"));
338 encoded_frame_dump_files.push_back(
339 FileWrapper::OpenWriteOnly(prefix + "3.ivf"));
340 }
Niels Möllerb7edf692019-02-08 15:40:53341
Erik Språng616b2332019-02-11 13:16:28342 double overshoot_factor = 1.0;
343 // Match format to either of the streams in dual-stream mode in order to get
344 // the overshoot factor. This is not very robust but we can't know for sure
345 // which stream this encoder is meant for, from within the factory.
346 if (format ==
347 SdpVideoFormat(params_.video[0].codec, params_.video[0].sdp_params)) {
348 overshoot_factor = params_.video[0].encoder_overshoot_factor;
349 } else if (format == SdpVideoFormat(params_.video[1].codec,
350 params_.video[1].sdp_params)) {
351 overshoot_factor = params_.video[1].encoder_overshoot_factor;
352 }
353 if (overshoot_factor == 0.0) {
354 // If params were zero-initialized, set to 1.0 instead.
355 overshoot_factor = 1.0;
356 }
357
358 if (analyzer || !encoded_frame_dump_files.empty() || overshoot_factor > 1.0) {
Mirko Bonadei317a1f02019-09-17 15:06:18359 encoder = std::make_unique<QualityTestVideoEncoder>(
Erik Språng616b2332019-02-11 13:16:28360 std::move(encoder), analyzer, std::move(encoded_frame_dump_files),
361 overshoot_factor);
Niels Möller1beef1a2018-09-03 12:21:44362 }
Niels Möller88be9722018-10-10 08:58:52363
Niels Möller1beef1a2018-09-03 12:21:44364 return encoder;
Niels Möller4db138e2018-04-19 07:04:13365}
366
Patrik Höglundb6b29e02018-06-21 14:58:01367VideoQualityTest::VideoQualityTest(
Artem Titove269cb42018-08-29 07:59:23368 std::unique_ptr<InjectionComponents> injection_components)
Sebastian Janssone6d7c3e2018-07-11 13:00:41369 : clock_(Clock::GetRealTimeClock()),
Danil Chapovalov1c41be62019-04-01 07:16:12370 task_queue_factory_(CreateDefaultTaskQueueFactory()),
Danil Chapovalov61b1f532024-01-29 09:36:53371 video_decoder_factory_(
372 [this](const Environment& env, const SdpVideoFormat& format) {
373 return this->CreateVideoDecoder(env, format);
374 }),
Danil Chapovalov329f0ea2024-03-08 13:30:54375 video_encoder_factory_(
376 [this](const Environment& env, const SdpVideoFormat& format) {
377 return this->CreateVideoEncoder(env, format, nullptr);
378 }),
Niels Möller88be9722018-10-10 08:58:52379 video_encoder_factory_with_analyzer_(
Danil Chapovalov329f0ea2024-03-08 13:30:54380 [this](const Environment& env, const SdpVideoFormat& format) {
381 return this->CreateVideoEncoder(env, format, analyzer_.get());
Niels Möller88be9722018-10-10 08:58:52382 }),
Jiawei Ouc2ebe212018-11-08 18:02:56383 video_bitrate_allocator_factory_(
384 CreateBuiltinVideoBitrateAllocatorFactory()),
Sebastian Janssone6d7c3e2018-07-11 13:00:41385 receive_logs_(0),
Artem Titove269cb42018-08-29 07:59:23386 send_logs_(0),
Erik Språng96965ae2018-10-23 13:42:37387 injection_components_(std::move(injection_components)),
388 num_video_streams_(0) {
Artem Titove269cb42018-08-29 07:59:23389 if (injection_components_ == nullptr) {
Mirko Bonadei317a1f02019-09-17 15:06:18390 injection_components_ = std::make_unique<InjectionComponents>();
Artem Titove269cb42018-08-29 07:59:23391 }
“Michael3c396e52019-06-12 14:47:14392 if (injection_components_->video_decoder_factory != nullptr) {
393 decoder_factory_ = std::move(injection_components_->video_decoder_factory);
394 } else {
Mirko Bonadei317a1f02019-09-17 15:06:18395 decoder_factory_ = std::make_unique<InternalDecoderFactory>();
“Michael3c396e52019-06-12 14:47:14396 }
397 if (injection_components_->video_encoder_factory != nullptr) {
398 encoder_factory_ = std::move(injection_components_->video_encoder_factory);
399 } else {
Mirko Bonadei317a1f02019-09-17 15:06:18400 encoder_factory_ = std::make_unique<InternalEncoderFactory>();
“Michael3c396e52019-06-12 14:47:14401 }
Artem Titove269cb42018-08-29 07:59:23402
Artem Titove269cb42018-08-29 07:59:23403 fec_controller_factory_ =
404 std::move(injection_components_->fec_controller_factory);
Ying Wangcab77fd2019-04-16 09:12:49405 network_state_predictor_factory_ =
406 std::move(injection_components_->network_state_predictor_factory);
Sebastian Jansson1391ed22019-04-30 12:23:51407 network_controller_factory_ =
408 std::move(injection_components_->network_controller_factory);
Per Kjellander89870ff2023-01-19 15:45:58409
410 // Register header extensions that are used by transport to identify
411 // extensions when parsing incomig packets.
412 RegisterRtpExtension(RtpExtension(RtpExtension::kTransportSequenceNumberUri,
413 kTransportSequenceNumberExtensionId));
414 RegisterRtpExtension(
415 RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId));
416 RegisterRtpExtension(RtpExtension(RtpExtension::kGenericFrameDescriptorUri00,
417 kGenericFrameDescriptorExtensionId00));
418 RegisterRtpExtension(RtpExtension(RtpExtension::kDependencyDescriptorUri,
419 kRtpExtensionDependencyDescriptor));
420 RegisterRtpExtension(RtpExtension(RtpExtension::kVideoContentTypeUri,
421 kVideoContentTypeExtensionId));
422 RegisterRtpExtension(
423 RtpExtension(RtpExtension::kVideoTimingUri, kVideoTimingExtensionId));
Ying Wang3b790f32018-01-19 16:58:57424}
425
Artem Titove269cb42018-08-29 07:59:23426VideoQualityTest::InjectionComponents::InjectionComponents() = default;
427
428VideoQualityTest::InjectionComponents::~InjectionComponents() = default;
429
ivica5d6a06c2015-09-17 12:30:24430void VideoQualityTest::TestBody() {}
431
sprangce4aef12015-11-02 15:23:20432std::string VideoQualityTest::GenerateGraphTitle() const {
Jonas Olsson366a50c2018-09-06 11:41:30433 rtc::StringBuilder ss;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59434 ss << params_.video[0].codec;
435 ss << " (" << params_.video[0].target_bitrate_bps / 1000 << "kbps";
436 ss << ", " << params_.video[0].fps << " FPS";
437 if (params_.screenshare[0].scroll_duration)
438 ss << ", " << params_.screenshare[0].scroll_duration << "s scroll";
439 if (params_.ss[0].streams.size() > 1)
440 ss << ", Stream #" << params_.ss[0].selected_stream;
441 if (params_.ss[0].num_spatial_layers > 1)
442 ss << ", Layer #" << params_.ss[0].selected_sl;
sprangce4aef12015-11-02 15:23:20443 ss << ")";
Jonas Olsson84df1c72018-09-14 14:59:32444 return ss.Release();
sprangce4aef12015-11-02 15:23:20445}
446
Artem Titove269cb42018-08-29 07:59:23447void VideoQualityTest::CheckParamsAndInjectionComponents() {
448 if (injection_components_ == nullptr) {
Mirko Bonadei317a1f02019-09-17 15:06:18449 injection_components_ = std::make_unique<InjectionComponents>();
Artem Titove269cb42018-08-29 07:59:23450 }
451 if (!params_.config && injection_components_->sender_network == nullptr &&
452 injection_components_->receiver_network == nullptr) {
Artem Titov62ae1782018-10-24 12:14:49453 params_.config = BuiltInNetworkBehaviorConfig();
Artem Titovf18b3522018-08-28 14:54:24454 }
Artem Titove269cb42018-08-29 07:59:23455 RTC_CHECK(
456 (params_.config && injection_components_->sender_network == nullptr &&
457 injection_components_->receiver_network == nullptr) ||
458 (!params_.config && injection_components_->sender_network != nullptr &&
459 injection_components_->receiver_network != nullptr));
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59460 for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) {
461 // Iterate over primary and secondary video streams.
462 if (!params_.video[video_idx].enabled)
463 return;
464 // Add a default stream in none specified.
465 if (params_.ss[video_idx].streams.empty())
466 params_.ss[video_idx].streams.push_back(
467 VideoQualityTest::DefaultVideoStream(params_, video_idx));
468 if (params_.ss[video_idx].num_spatial_layers == 0)
469 params_.ss[video_idx].num_spatial_layers = 1;
sprangce4aef12015-11-02 15:23:20470
Artem Titove269cb42018-08-29 07:59:23471 if (params_.config) {
472 if (params_.config->loss_percent != 0 ||
473 params_.config->queue_length_packets != 0) {
474 // Since LayerFilteringTransport changes the sequence numbers, we can't
475 // use that feature with pack loss, since the NACK request would end up
476 // retransmitting the wrong packets.
477 RTC_CHECK(params_.ss[video_idx].selected_sl == -1 ||
478 params_.ss[video_idx].selected_sl ==
479 params_.ss[video_idx].num_spatial_layers - 1);
480 RTC_CHECK(params_.video[video_idx].selected_tl == -1 ||
481 params_.video[video_idx].selected_tl ==
482 params_.video[video_idx].num_temporal_layers - 1);
483 }
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59484 }
sprangce4aef12015-11-02 15:23:20485
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59486 // TODO(ivica): Should max_bitrate_bps == -1 represent inf max bitrate, as
487 // it does in some parts of the code?
488 RTC_CHECK_GE(params_.video[video_idx].max_bitrate_bps,
489 params_.video[video_idx].target_bitrate_bps);
490 RTC_CHECK_GE(params_.video[video_idx].target_bitrate_bps,
491 params_.video[video_idx].min_bitrate_bps);
Ilya Nikolaevskiyaec663e2019-02-27 11:52:11492 int selected_stream = params_.ss[video_idx].selected_stream;
Evan Shrubsole420ad1a2020-03-04 09:42:28493 if (params_.video[video_idx].selected_tl > -1) {
494 RTC_CHECK_LT(selected_stream, params_.ss[video_idx].streams.size())
495 << "Can not use --selected_tl when --selected_stream is all streams";
496 int stream_tl = params_.ss[video_idx]
497 .streams[selected_stream]
498 .num_temporal_layers.value_or(1);
499 RTC_CHECK_LT(params_.video[video_idx].selected_tl, stream_tl);
500 }
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59501 RTC_CHECK_LE(params_.ss[video_idx].selected_stream,
502 params_.ss[video_idx].streams.size());
503 for (const VideoStream& stream : params_.ss[video_idx].streams) {
504 RTC_CHECK_GE(stream.min_bitrate_bps, 0);
505 RTC_CHECK_GE(stream.target_bitrate_bps, stream.min_bitrate_bps);
506 RTC_CHECK_GE(stream.max_bitrate_bps, stream.target_bitrate_bps);
507 }
508 // TODO(ivica): Should we check if the sum of all streams/layers is equal to
509 // the total bitrate? We anyway have to update them in the case bitrate
510 // estimator changes the total bitrates.
511 RTC_CHECK_GE(params_.ss[video_idx].num_spatial_layers, 1);
512 RTC_CHECK_LE(params_.ss[video_idx].selected_sl,
513 params_.ss[video_idx].num_spatial_layers);
514 RTC_CHECK(
515 params_.ss[video_idx].spatial_layers.empty() ||
516 params_.ss[video_idx].spatial_layers.size() ==
517 static_cast<size_t>(params_.ss[video_idx].num_spatial_layers));
518 if (params_.video[video_idx].codec == "VP8") {
519 RTC_CHECK_EQ(params_.ss[video_idx].num_spatial_layers, 1);
520 } else if (params_.video[video_idx].codec == "VP9") {
521 RTC_CHECK_EQ(params_.ss[video_idx].streams.size(), 1);
522 }
523 RTC_CHECK_GE(params_.call.num_thumbnails, 0);
524 if (params_.call.num_thumbnails > 0) {
525 RTC_CHECK_EQ(params_.ss[video_idx].num_spatial_layers, 1);
526 RTC_CHECK_EQ(params_.ss[video_idx].streams.size(), 3);
527 RTC_CHECK_EQ(params_.video[video_idx].num_temporal_layers, 3);
528 RTC_CHECK_EQ(params_.video[video_idx].codec, "VP8");
529 }
530 // Dual streams with FEC not supported in tests yet.
531 RTC_CHECK(!params_.video[video_idx].flexfec || num_video_streams_ == 1);
532 RTC_CHECK(!params_.video[video_idx].ulpfec || num_video_streams_ == 1);
ilnika014cc52017-03-07 12:21:04533 }
sprangce4aef12015-11-02 15:23:20534}
535
536// Static.
537std::vector<int> VideoQualityTest::ParseCSV(const std::string& str) {
538 // Parse comma separated nonnegative integers, where some elements may be
539 // empty. The empty values are replaced with -1.
540 // E.g. "10,-20,,30,40" --> {10, 20, -1, 30,40}
541 // E.g. ",,10,,20," --> {-1, -1, 10, -1, 20, -1}
542 std::vector<int> result;
543 if (str.empty())
544 return result;
545
546 const char* p = str.c_str();
547 int value = -1;
548 int pos;
549 while (*p) {
550 if (*p == ',') {
551 result.push_back(value);
552 value = -1;
553 ++p;
554 continue;
555 }
556 RTC_CHECK_EQ(sscanf(p, "%d%n", &value, &pos), 1)
557 << "Unexpected non-number value.";
558 p += pos;
559 }
560 result.push_back(value);
561 return result;
562}
563
564// Static.
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59565VideoStream VideoQualityTest::DefaultVideoStream(const Params& params,
566 size_t video_idx) {
sprangce4aef12015-11-02 15:23:20567 VideoStream stream;
Danil Chapovalov350531e2018-06-08 11:04:04568 stream.width = params.video[video_idx].width;
569 stream.height = params.video[video_idx].height;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59570 stream.max_framerate = params.video[video_idx].fps;
571 stream.min_bitrate_bps = params.video[video_idx].min_bitrate_bps;
572 stream.target_bitrate_bps = params.video[video_idx].target_bitrate_bps;
573 stream.max_bitrate_bps = params.video[video_idx].max_bitrate_bps;
Sergey Silkinb6ef1a72023-10-23 15:11:21574 stream.max_qp = cricket::kDefaultVideoMaxQpVpx;
Sergey Silkina796a7e2018-03-01 14:11:29575 stream.num_temporal_layers = params.video[video_idx].num_temporal_layers;
Seth Hampson46e31ba2018-01-18 18:39:54576 stream.active = true;
ilnika014cc52017-03-07 12:21:04577 return stream;
578}
579
580// Static.
581VideoStream VideoQualityTest::DefaultThumbnailStream() {
582 VideoStream stream;
583 stream.width = 320;
584 stream.height = 180;
585 stream.max_framerate = 7;
586 stream.min_bitrate_bps = 7500;
587 stream.target_bitrate_bps = 37500;
588 stream.max_bitrate_bps = 50000;
Sergey Silkinb6ef1a72023-10-23 15:11:21589 stream.max_qp = cricket::kDefaultVideoMaxQpVpx;
sprangce4aef12015-11-02 15:23:20590 return stream;
591}
592
sprangce4aef12015-11-02 15:23:20593void VideoQualityTest::FillScalabilitySettings(
594 Params* params,
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59595 size_t video_idx,
sprangce4aef12015-11-02 15:23:20596 const std::vector<std::string>& stream_descriptors,
sprang1168fd42017-06-21 16:00:17597 int num_streams,
sprangce4aef12015-11-02 15:23:20598 size_t selected_stream,
599 int num_spatial_layers,
600 int selected_sl,
Sergey Silkin57027362018-05-15 07:12:05601 InterLayerPredMode inter_layer_pred,
sprangce4aef12015-11-02 15:23:20602 const std::vector<std::string>& sl_descriptors) {
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59603 if (params->ss[video_idx].streams.empty() &&
604 params->ss[video_idx].infer_streams) {
Jonas Oreland80c87d72022-09-29 13:01:09605 webrtc::VideoEncoder::EncoderInfo encoder_info;
sprang1168fd42017-06-21 16:00:17606 webrtc::VideoEncoderConfig encoder_config;
Niels Möller259a4972018-04-05 13:36:51607 encoder_config.codec_type =
608 PayloadStringToCodecType(params->video[video_idx].codec);
sprang1168fd42017-06-21 16:00:17609 encoder_config.content_type =
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59610 params->screenshare[video_idx].enabled
sprang1168fd42017-06-21 16:00:17611 ? webrtc::VideoEncoderConfig::ContentType::kScreen
612 : webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59613 encoder_config.max_bitrate_bps = params->video[video_idx].max_bitrate_bps;
614 encoder_config.min_transmit_bitrate_bps =
615 params->video[video_idx].min_transmit_bps;
sprang1168fd42017-06-21 16:00:17616 encoder_config.number_of_streams = num_streams;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59617 encoder_config.spatial_layers = params->ss[video_idx].spatial_layers;
Seth Hampson8234ead2018-02-02 23:16:24618 encoder_config.simulcast_layers = std::vector<VideoStream>(num_streams);
sprang1168fd42017-06-21 16:00:17619 encoder_config.video_stream_factory =
Sergey Silkin3172d162024-07-08 11:57:52620 rtc::make_ref_counted<cricket::EncoderStreamFactory>(encoder_info);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59621 params->ss[video_idx].streams =
sprang1168fd42017-06-21 16:00:17622 encoder_config.video_stream_factory->CreateEncoderStreams(
Danil Chapovalov44ab2002024-04-17 10:11:29623 env().field_trials(), params->video[video_idx].width,
624 params->video[video_idx].height, encoder_config);
sprang1168fd42017-06-21 16:00:17625 } else {
626 // Read VideoStream and SpatialLayer elements from a list of comma separated
627 // lists. To use a default value for an element, use -1 or leave empty.
Artem Titove269cb42018-08-29 07:59:23628 // Validity checks performed in CheckParamsAndInjectionComponents.
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59629 RTC_CHECK(params->ss[video_idx].streams.empty());
Mirko Bonadei739baf02019-01-27 16:29:42630 for (const auto& descriptor : stream_descriptors) {
sprang1168fd42017-06-21 16:00:17631 if (descriptor.empty())
632 continue;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59633 VideoStream stream =
634 VideoQualityTest::DefaultVideoStream(*params, video_idx);
sprang1168fd42017-06-21 16:00:17635 std::vector<int> v = VideoQualityTest::ParseCSV(descriptor);
636 if (v[0] != -1)
637 stream.width = static_cast<size_t>(v[0]);
638 if (v[1] != -1)
639 stream.height = static_cast<size_t>(v[1]);
640 if (v[2] != -1)
641 stream.max_framerate = v[2];
642 if (v[3] != -1)
643 stream.min_bitrate_bps = v[3];
644 if (v[4] != -1)
645 stream.target_bitrate_bps = v[4];
646 if (v[5] != -1)
647 stream.max_bitrate_bps = v[5];
648 if (v.size() > 6 && v[6] != -1)
649 stream.max_qp = v[6];
Sergey Silkina796a7e2018-03-01 14:11:29650 if (v.size() > 7 && v[7] != -1) {
651 stream.num_temporal_layers = v[7];
sprang1168fd42017-06-21 16:00:17652 } else {
653 // Automatic TL thresholds for more than two layers not supported.
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59654 RTC_CHECK_LE(params->video[video_idx].num_temporal_layers, 2);
sprang1168fd42017-06-21 16:00:17655 }
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59656 params->ss[video_idx].streams.push_back(stream);
sprangce4aef12015-11-02 15:23:20657 }
sprangce4aef12015-11-02 15:23:20658 }
sprangce4aef12015-11-02 15:23:20659
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59660 params->ss[video_idx].num_spatial_layers = std::max(1, num_spatial_layers);
661 params->ss[video_idx].selected_stream = selected_stream;
sprang1168fd42017-06-21 16:00:17662
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59663 params->ss[video_idx].selected_sl = selected_sl;
Sergey Silkin57027362018-05-15 07:12:05664 params->ss[video_idx].inter_layer_pred = inter_layer_pred;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59665 RTC_CHECK(params->ss[video_idx].spatial_layers.empty());
Mirko Bonadei739baf02019-01-27 16:29:42666 for (const auto& descriptor : sl_descriptors) {
sprangce4aef12015-11-02 15:23:20667 if (descriptor.empty())
668 continue;
669 std::vector<int> v = VideoQualityTest::ParseCSV(descriptor);
Sergey Silkina89800c2019-02-19 11:52:56670 RTC_CHECK_EQ(v.size(), 8);
sprangce4aef12015-11-02 15:23:20671
Sergey Silkin13e74342018-03-02 11:28:00672 SpatialLayer layer = {0};
673 layer.width = v[0];
674 layer.height = v[1];
Sergey Silkina89800c2019-02-19 11:52:56675 layer.maxFramerate = v[2];
676 layer.numberOfTemporalLayers = v[3];
677 layer.maxBitrate = v[4];
678 layer.minBitrate = v[5];
679 layer.targetBitrate = v[6];
680 layer.qpMax = v[7];
Sergey Silkin13e74342018-03-02 11:28:00681 layer.active = true;
682
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59683 params->ss[video_idx].spatial_layers.push_back(layer);
sprangce4aef12015-11-02 15:23:20684 }
685}
686
minyuea27172d2016-11-01 12:59:29687void VideoQualityTest::SetupVideo(Transport* send_transport,
688 Transport* recv_transport) {
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59689 size_t total_streams_used = 0;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59690 video_receive_configs_.clear();
691 video_send_configs_.clear();
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59692 video_encoder_configs_.clear();
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59693 bool decode_all_receive_streams = true;
694 size_t num_video_substreams = params_.ss[0].streams.size();
695 RTC_CHECK(num_video_streams_ > 0);
696 video_encoder_configs_.resize(num_video_streams_);
Rasmus Brandt5894b6a2019-06-13 14:28:14697 std::string generic_codec_name;
Jonas Oreland80c87d72022-09-29 13:01:09698 webrtc::VideoEncoder::EncoderInfo encoder_info;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59699 for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) {
philipel7496ff62022-12-30 12:21:07700 VideoSendStream::Config config(send_transport);
701 config.rtp.extmap_allow_mixed = true;
702 video_send_configs_.push_back(std::move(config));
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59703 video_encoder_configs_.push_back(VideoEncoderConfig());
704 num_video_substreams = params_.ss[video_idx].streams.size();
705 RTC_CHECK_GT(num_video_substreams, 0);
Sebastian Jansson3bd2c792018-07-13 11:29:03706 for (size_t i = 0; i < num_video_substreams; ++i)
707 video_send_configs_[video_idx].rtp.ssrcs.push_back(
Artem Titov8a9f3a82023-04-25 07:56:49708 test::VideoTestConstants::kVideoSendSsrcs[total_streams_used + i]);
Qingsi Wang2d82ade2018-07-11 06:04:44709
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59710 int payload_type;
711 if (params_.video[video_idx].codec == "H264") {
Artem Titov8a9f3a82023-04-25 07:56:49712 payload_type = test::VideoTestConstants::kPayloadTypeH264;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59713 } else if (params_.video[video_idx].codec == "VP8") {
Artem Titov8a9f3a82023-04-25 07:56:49714 payload_type = test::VideoTestConstants::kPayloadTypeVP8;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59715 } else if (params_.video[video_idx].codec == "VP9") {
Artem Titov8a9f3a82023-04-25 07:56:49716 payload_type = test::VideoTestConstants::kPayloadTypeVP9;
Emircan Uysaler7c03bdc2019-01-16 20:07:56717 } else if (params_.video[video_idx].codec == "FakeCodec") {
Artem Titov8a9f3a82023-04-25 07:56:49718 payload_type = test::VideoTestConstants::kFakeVideoSendPayloadType;
ilnikcb8c1462017-03-09 17:23:30719 } else {
Rasmus Brandt5894b6a2019-06-13 14:28:14720 RTC_CHECK(generic_codec_name.empty() ||
721 generic_codec_name == params_.video[video_idx].codec)
722 << "Supplying multiple generic codecs is unsupported.";
723 RTC_LOG(LS_INFO) << "Treating codec " << params_.video[video_idx].codec
724 << " as generic.";
Artem Titov8a9f3a82023-04-25 07:56:49725 payload_type = test::VideoTestConstants::kPayloadTypeGeneric;
Rasmus Brandt5894b6a2019-06-13 14:28:14726 generic_codec_name = params_.video[video_idx].codec;
ilnikcb8c1462017-03-09 17:23:30727 }
Niels Möller4db138e2018-04-19 07:04:13728 video_send_configs_[video_idx].encoder_settings.encoder_factory =
Niels Möller88be9722018-10-10 08:58:52729 (video_idx == 0) ? &video_encoder_factory_with_analyzer_
730 : &video_encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56731 video_send_configs_[video_idx].encoder_settings.bitrate_allocator_factory =
732 video_bitrate_allocator_factory_.get();
Niels Möller4db138e2018-04-19 07:04:13733
Niels Möller259a4972018-04-05 13:36:51734 video_send_configs_[video_idx].rtp.payload_name =
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59735 params_.video[video_idx].codec;
Niels Möller259a4972018-04-05 13:36:51736 video_send_configs_[video_idx].rtp.payload_type = payload_type;
Artem Titov8a9f3a82023-04-25 07:56:49737 video_send_configs_[video_idx].rtp.nack.rtp_history_ms =
738 test::VideoTestConstants::kNackRtpHistoryMs;
739 video_send_configs_[video_idx].rtp.rtx.payload_type =
740 test::VideoTestConstants::kSendRtxPayloadType;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59741 for (size_t i = 0; i < num_video_substreams; ++i) {
742 video_send_configs_[video_idx].rtp.rtx.ssrcs.push_back(
Artem Titov8a9f3a82023-04-25 07:56:49743 test::VideoTestConstants::kSendRtxSsrcs[i + total_streams_used]);
ilnik9fd9f6c2017-03-02 16:10:10744 }
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59745 video_send_configs_[video_idx].rtp.extensions.clear();
746 if (params_.call.send_side_bwe) {
philipel569397f2018-09-26 10:25:31747 video_send_configs_[video_idx].rtp.extensions.emplace_back(
748 RtpExtension::kTransportSequenceNumberUri,
Elad Alond8d32482019-02-18 22:45:57749 kTransportSequenceNumberExtensionId);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59750 } else {
philipel569397f2018-09-26 10:25:31751 video_send_configs_[video_idx].rtp.extensions.emplace_back(
Elad Alond8d32482019-02-18 22:45:57752 RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59753 }
philipel569397f2018-09-26 10:25:31754
755 if (params_.call.generic_descriptor) {
philipel569397f2018-09-26 10:25:31756 video_send_configs_[video_idx].rtp.extensions.emplace_back(
Elad Alonccb9b752019-02-19 12:01:31757 RtpExtension::kGenericFrameDescriptorUri00,
758 kGenericFrameDescriptorExtensionId00);
philipel569397f2018-09-26 10:25:31759 }
760
philipel7496ff62022-12-30 12:21:07761 if (params_.call.dependency_descriptor) {
762 video_send_configs_[video_idx].rtp.extensions.emplace_back(
763 RtpExtension::kDependencyDescriptorUri,
philipel0dbce832023-01-03 08:43:53764 kRtpExtensionDependencyDescriptor);
philipel7496ff62022-12-30 12:21:07765 }
766
philipel569397f2018-09-26 10:25:31767 video_send_configs_[video_idx].rtp.extensions.emplace_back(
Elad Alond8d32482019-02-18 22:45:57768 RtpExtension::kVideoContentTypeUri, kVideoContentTypeExtensionId);
philipel569397f2018-09-26 10:25:31769 video_send_configs_[video_idx].rtp.extensions.emplace_back(
Elad Alond8d32482019-02-18 22:45:57770 RtpExtension::kVideoTimingUri, kVideoTimingExtensionId);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59771
Niels Möller4db138e2018-04-19 07:04:13772 video_encoder_configs_[video_idx].video_format.name =
773 params_.video[video_idx].codec;
774
Emircan Uysaler0823eec2018-07-14 00:10:00775 video_encoder_configs_[video_idx].video_format.parameters =
776 params_.video[video_idx].sdp_params;
777
Niels Möller259a4972018-04-05 13:36:51778 video_encoder_configs_[video_idx].codec_type =
779 PayloadStringToCodecType(params_.video[video_idx].codec);
780
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59781 video_encoder_configs_[video_idx].min_transmit_bitrate_bps =
782 params_.video[video_idx].min_transmit_bps;
783
784 video_send_configs_[video_idx].suspend_below_min_bitrate =
785 params_.video[video_idx].suspend_below_min_bitrate;
786
787 video_encoder_configs_[video_idx].number_of_streams =
788 params_.ss[video_idx].streams.size();
789 video_encoder_configs_[video_idx].max_bitrate_bps = 0;
790 for (size_t i = 0; i < params_.ss[video_idx].streams.size(); ++i) {
791 video_encoder_configs_[video_idx].max_bitrate_bps +=
792 params_.ss[video_idx].streams[i].max_bitrate_bps;
793 }
Ilya Nikolaevskiy6957abe2019-01-29 15:33:04794 video_encoder_configs_[video_idx].simulcast_layers =
795 std::vector<VideoStream>(params_.ss[video_idx].streams.size());
796 if (!params_.ss[video_idx].infer_streams) {
Seth Hampson8234ead2018-02-02 23:16:24797 video_encoder_configs_[video_idx].simulcast_layers =
Ilya Nikolaevskiy6957abe2019-01-29 15:33:04798 params_.ss[video_idx].streams;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59799 }
800
801 video_encoder_configs_[video_idx].spatial_layers =
802 params_.ss[video_idx].spatial_layers;
Niels Möller8139cda2022-05-19 13:54:13803
804 video_encoder_configs_[video_idx].frame_drop_enabled = true;
805
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59806 decode_all_receive_streams = params_.ss[video_idx].selected_stream ==
807 params_.ss[video_idx].streams.size();
Sebastian Jansson3bd2c792018-07-13 11:29:03808 absl::optional<int> decode_sub_stream;
809 if (!decode_all_receive_streams)
810 decode_sub_stream = params_.ss[video_idx].selected_stream;
811 CreateMatchingVideoReceiveConfigs(
Per K92532402023-01-02 15:57:18812 video_send_configs_[video_idx], recv_transport, &video_decoder_factory_,
Artem Titov8a9f3a82023-04-25 07:56:49813 decode_sub_stream, true, test::VideoTestConstants::kNackRtpHistoryMs);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59814
815 if (params_.screenshare[video_idx].enabled) {
816 // Fill out codec settings.
817 video_encoder_configs_[video_idx].content_type =
818 VideoEncoderConfig::ContentType::kScreen;
Sergey Silkin3172d162024-07-08 11:57:52819 video_encoder_configs_[video_idx].legacy_conference_mode = true;
Taylor Brandstetter49fcc102018-05-16 21:20:41820 degradation_preference_ = DegradationPreference::MAINTAIN_RESOLUTION;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59821 if (params_.video[video_idx].codec == "VP8") {
822 VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
823 vp8_settings.denoisingOn = false;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59824 vp8_settings.numberOfTemporalLayers = static_cast<unsigned char>(
825 params_.video[video_idx].num_temporal_layers);
826 video_encoder_configs_[video_idx].encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43827 rtc::make_ref_counted<
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59828 VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
829 } else if (params_.video[video_idx].codec == "VP9") {
830 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
831 vp9_settings.denoisingOn = false;
Ilya Nikolaevskiyeac08bf2020-03-10 09:50:26832 vp9_settings.automaticResizeOn = false;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59833 vp9_settings.numberOfTemporalLayers = static_cast<unsigned char>(
834 params_.video[video_idx].num_temporal_layers);
835 vp9_settings.numberOfSpatialLayers = static_cast<unsigned char>(
836 params_.ss[video_idx].num_spatial_layers);
Sergey Silkin57027362018-05-15 07:12:05837 vp9_settings.interLayerPred = params_.ss[video_idx].inter_layer_pred;
Ilya Nikolaevskiy5546aef2018-12-04 14:54:52838 // High FPS vp9 screenshare requires flexible mode.
Ilya Nikolaevskiy5c18a5f2019-05-24 09:51:31839 if (params_.ss[video_idx].num_spatial_layers > 1) {
Ilya Nikolaevskiy5546aef2018-12-04 14:54:52840 vp9_settings.flexibleMode = true;
841 }
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59842 video_encoder_configs_[video_idx].encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43843 rtc::make_ref_counted<
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59844 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
845 }
846 } else if (params_.ss[video_idx].num_spatial_layers > 1) {
847 // If SVC mode without screenshare, still need to set codec specifics.
848 RTC_CHECK(params_.video[video_idx].codec == "VP9");
849 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
850 vp9_settings.numberOfTemporalLayers = static_cast<unsigned char>(
851 params_.video[video_idx].num_temporal_layers);
852 vp9_settings.numberOfSpatialLayers =
853 static_cast<unsigned char>(params_.ss[video_idx].num_spatial_layers);
Sergey Silkin57027362018-05-15 07:12:05854 vp9_settings.interLayerPred = params_.ss[video_idx].inter_layer_pred;
Ilya Nikolaevskiyeac08bf2020-03-10 09:50:26855 vp9_settings.automaticResizeOn = false;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59856 video_encoder_configs_[video_idx].encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43857 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
858 vp9_settings);
Erik Språng25a13342020-10-27 12:01:37859 RTC_DCHECK_EQ(video_encoder_configs_[video_idx].simulcast_layers.size(),
860 1);
861 // Min bitrate will be enforced by spatial layer config instead.
862 video_encoder_configs_[video_idx].simulcast_layers[0].min_bitrate_bps = 0;
Niels Möller6aa415e2018-06-07 09:14:13863 } else if (params_.video[video_idx].automatic_scaling) {
864 if (params_.video[video_idx].codec == "VP8") {
865 VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
866 vp8_settings.automaticResizeOn = true;
867 video_encoder_configs_[video_idx].encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43868 rtc::make_ref_counted<
Niels Möller6aa415e2018-06-07 09:14:13869 VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
870 } else if (params_.video[video_idx].codec == "VP9") {
871 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
Ilya Nikolaevskiyeac08bf2020-03-10 09:50:26872 // Only enable quality scaler for single spatial layer.
873 vp9_settings.automaticResizeOn =
874 params_.ss[video_idx].num_spatial_layers == 1;
Niels Möller6aa415e2018-06-07 09:14:13875 video_encoder_configs_[video_idx].encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43876 rtc::make_ref_counted<
Niels Möller6aa415e2018-06-07 09:14:13877 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
Stefan Holmer1f7a0082019-01-11 14:39:08878 } else if (params_.video[video_idx].codec == "H264") {
879 // Quality scaling is always on for H.264.
Danil Chapovalova54ba4c2020-05-11 10:36:13880 } else if (params_.video[video_idx].codec == cricket::kAv1CodecName) {
881 // TODO(bugs.webrtc.org/11404): Propagate the flag to
882 // aom_codec_enc_cfg_t::rc_resize_mode in Av1 encoder wrapper.
883 // Until then do nothing, specially do not crash.
Niels Möller6aa415e2018-06-07 09:14:13884 } else {
Artem Titovd3251962021-11-15 15:57:07885 RTC_DCHECK_NOTREACHED()
886 << "Automatic scaling not supported for codec "
887 << params_.video[video_idx].codec << ", stream " << video_idx;
Niels Möller6aa415e2018-06-07 09:14:13888 }
Ilya Nikolaevskiy6957abe2019-01-29 15:33:04889 } else {
890 // Default mode. Single SL, no automatic_scaling,
891 if (params_.video[video_idx].codec == "VP8") {
892 VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
893 vp8_settings.automaticResizeOn = false;
894 video_encoder_configs_[video_idx].encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43895 rtc::make_ref_counted<
Ilya Nikolaevskiy6957abe2019-01-29 15:33:04896 VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
897 } else if (params_.video[video_idx].codec == "VP9") {
898 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
899 vp9_settings.automaticResizeOn = false;
900 video_encoder_configs_[video_idx].encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43901 rtc::make_ref_counted<
Ilya Nikolaevskiy6957abe2019-01-29 15:33:04902 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
903 } else if (params_.video[video_idx].codec == "H264") {
Niels Möllercf2c8912022-05-18 08:45:46904 video_encoder_configs_[video_idx].encoder_specific_settings = nullptr;
Ilya Nikolaevskiy6957abe2019-01-29 15:33:04905 }
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59906 }
907 total_streams_used += num_video_substreams;
sprangce4aef12015-11-02 15:23:20908 }
brandtr1293aca2016-11-17 06:47:29909
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59910 // FEC supported only for single video stream mode yet.
911 if (params_.video[0].flexfec) {
sprang1168fd42017-06-21 16:00:17912 if (decode_all_receive_streams) {
Sebastian Jansson3bd2c792018-07-13 11:29:03913 SetSendFecConfig(GetVideoSendConfig()->rtp.ssrcs);
sprang1168fd42017-06-21 16:00:17914 } else {
Artem Titov8a9f3a82023-04-25 07:56:49915 SetSendFecConfig(
916 {test::VideoTestConstants::kVideoSendSsrcs[params_.ss[0]
917 .selected_stream]});
sprang1168fd42017-06-21 16:00:17918 }
brandtr1293aca2016-11-17 06:47:29919
Sebastian Jansson3bd2c792018-07-13 11:29:03920 CreateMatchingFecConfig(recv_transport, *GetVideoSendConfig());
brandtr1293aca2016-11-17 06:47:29921 }
922
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59923 if (params_.video[0].ulpfec) {
Sebastian Jansson3bd2c792018-07-13 11:29:03924 SetSendUlpFecConfig(GetVideoSendConfig());
sprang1168fd42017-06-21 16:00:17925 if (decode_all_receive_streams) {
Sebastian Jansson3bd2c792018-07-13 11:29:03926 for (auto& receive_config : video_receive_configs_) {
927 SetReceiveUlpFecConfig(&receive_config);
sprang1168fd42017-06-21 16:00:17928 }
929 } else {
Sebastian Jansson3bd2c792018-07-13 11:29:03930 SetReceiveUlpFecConfig(
931 &video_receive_configs_[params_.ss[0].selected_stream]);
sprang1168fd42017-06-21 16:00:17932 }
brandtr1293aca2016-11-17 06:47:29933 }
ivica5d6a06c2015-09-17 12:30:24934}
935
ilnika014cc52017-03-07 12:21:04936void VideoQualityTest::SetupThumbnails(Transport* send_transport,
937 Transport* recv_transport) {
ilnik98436952017-07-13 07:47:03938 for (int i = 0; i < params_.call.num_thumbnails; ++i) {
ilnika014cc52017-03-07 12:21:04939 // Thumbnails will be send in the other way: from receiver_call to
940 // sender_call.
941 VideoSendStream::Config thumbnail_send_config(recv_transport);
942 thumbnail_send_config.rtp.ssrcs.push_back(kThumbnailSendSsrcStart + i);
Niels Möller4db138e2018-04-19 07:04:13943 thumbnail_send_config.encoder_settings.encoder_factory =
944 &video_encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56945 thumbnail_send_config.encoder_settings.bitrate_allocator_factory =
946 video_bitrate_allocator_factory_.get();
Niels Möller259a4972018-04-05 13:36:51947 thumbnail_send_config.rtp.payload_name = params_.video[0].codec;
Artem Titov8a9f3a82023-04-25 07:56:49948 thumbnail_send_config.rtp.payload_type =
949 test::VideoTestConstants::kPayloadTypeVP8;
950 thumbnail_send_config.rtp.nack.rtp_history_ms =
951 test::VideoTestConstants::kNackRtpHistoryMs;
952 thumbnail_send_config.rtp.rtx.payload_type =
953 test::VideoTestConstants::kSendRtxPayloadType;
ilnika014cc52017-03-07 12:21:04954 thumbnail_send_config.rtp.rtx.ssrcs.push_back(kThumbnailRtxSsrcStart + i);
955 thumbnail_send_config.rtp.extensions.clear();
956 if (params_.call.send_side_bwe) {
957 thumbnail_send_config.rtp.extensions.push_back(
958 RtpExtension(RtpExtension::kTransportSequenceNumberUri,
Elad Alond8d32482019-02-18 22:45:57959 kTransportSequenceNumberExtensionId));
ilnika014cc52017-03-07 12:21:04960 } else {
Elad Alond8d32482019-02-18 22:45:57961 thumbnail_send_config.rtp.extensions.push_back(
962 RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId));
ilnika014cc52017-03-07 12:21:04963 }
964
965 VideoEncoderConfig thumbnail_encoder_config;
Niels Möller259a4972018-04-05 13:36:51966 thumbnail_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller4db138e2018-04-19 07:04:13967 thumbnail_encoder_config.video_format.name = "VP8";
ilnika014cc52017-03-07 12:21:04968 thumbnail_encoder_config.min_transmit_bitrate_bps = 7500;
969 thumbnail_send_config.suspend_below_min_bitrate =
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59970 params_.video[0].suspend_below_min_bitrate;
ilnika014cc52017-03-07 12:21:04971 thumbnail_encoder_config.number_of_streams = 1;
972 thumbnail_encoder_config.max_bitrate_bps = 50000;
Ilya Nikolaevskiy6957abe2019-01-29 15:33:04973 std::vector<VideoStream> streams{params_.ss[0].streams[0]};
974 thumbnail_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:43975 rtc::make_ref_counted<VideoStreamFactory>(streams);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59976 thumbnail_encoder_config.spatial_layers = params_.ss[0].spatial_layers;
ilnika014cc52017-03-07 12:21:04977
ilnika014cc52017-03-07 12:21:04978 thumbnail_encoder_configs_.push_back(thumbnail_encoder_config.Copy());
979 thumbnail_send_configs_.push_back(thumbnail_send_config.Copy());
ilnika014cc52017-03-07 12:21:04980
Sebastian Jansson3bd2c792018-07-13 11:29:03981 AddMatchingVideoReceiveConfigs(
982 &thumbnail_receive_configs_, thumbnail_send_config, send_transport,
Artem Titov8a9f3a82023-04-25 07:56:49983 &video_decoder_factory_, absl::nullopt, false,
984 test::VideoTestConstants::kNackRtpHistoryMs);
Sebastian Jansson3bd2c792018-07-13 11:29:03985 }
986 for (size_t i = 0; i < thumbnail_send_configs_.size(); ++i) {
ilnika014cc52017-03-07 12:21:04987 thumbnail_send_streams_.push_back(receiver_call_->CreateVideoSendStream(
988 thumbnail_send_configs_[i].Copy(),
989 thumbnail_encoder_configs_[i].Copy()));
Sebastian Jansson3bd2c792018-07-13 11:29:03990 }
991 for (size_t i = 0; i < thumbnail_receive_configs_.size(); ++i) {
ilnika014cc52017-03-07 12:21:04992 thumbnail_receive_streams_.push_back(sender_call_->CreateVideoReceiveStream(
993 thumbnail_receive_configs_[i].Copy()));
994 }
995}
996
997void VideoQualityTest::DestroyThumbnailStreams() {
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:59998 for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_) {
ilnika014cc52017-03-07 12:21:04999 receiver_call_->DestroyVideoSendStream(thumbnail_send_stream);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591000 }
ilnika014cc52017-03-07 12:21:041001 thumbnail_send_streams_.clear();
Tommif6f45432022-05-20 13:21:201002 for (VideoReceiveStreamInterface* thumbnail_receive_stream :
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591003 thumbnail_receive_streams_) {
ilnika014cc52017-03-07 12:21:041004 sender_call_->DestroyVideoReceiveStream(thumbnail_receive_stream);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591005 }
ilnika014cc52017-03-07 12:21:041006 thumbnail_send_streams_.clear();
1007 thumbnail_receive_streams_.clear();
Niels Möller1c931c42018-12-18 15:08:111008 for (std::unique_ptr<rtc::VideoSourceInterface<VideoFrame>>& video_capturer :
eladalon413ee9a2017-08-22 11:02:521009 thumbnail_capturers_) {
Niels Möller1c931c42018-12-18 15:08:111010 video_capturer.reset();
eladalon413ee9a2017-08-22 11:02:521011 }
ilnika014cc52017-03-07 12:21:041012}
1013
ilnika014cc52017-03-07 12:21:041014void VideoQualityTest::SetupThumbnailCapturers(size_t num_thumbnail_streams) {
1015 VideoStream thumbnail = DefaultThumbnailStream();
1016 for (size_t i = 0; i < num_thumbnail_streams; ++i) {
Danil Chapovalov908e22e2019-04-18 12:34:161017 auto frame_generator_capturer =
Mirko Bonadei317a1f02019-09-17 15:06:181018 std::make_unique<test::FrameGeneratorCapturer>(
Danil Chapovalov908e22e2019-04-18 12:34:161019 clock_,
Artem Titov33f9d2b2019-12-05 14:59:001020 test::CreateSquareFrameGenerator(static_cast<int>(thumbnail.width),
1021 static_cast<int>(thumbnail.height),
1022 absl::nullopt, absl::nullopt),
Danil Chapovalov908e22e2019-04-18 12:34:161023 thumbnail.max_framerate, *task_queue_factory_);
1024 EXPECT_TRUE(frame_generator_capturer->Init());
1025 thumbnail_capturers_.push_back(std::move(frame_generator_capturer));
ilnika014cc52017-03-07 12:21:041026 }
1027}
1028
Artem Titov33f9d2b2019-12-05 14:59:001029std::unique_ptr<test::FrameGeneratorInterface>
1030VideoQualityTest::CreateFrameGenerator(size_t video_idx) {
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591031 // Setup frame generator.
1032 const size_t kWidth = 1850;
1033 const size_t kHeight = 1110;
Artem Titov33f9d2b2019-12-05 14:59:001034 std::unique_ptr<test::FrameGeneratorInterface> frame_generator;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591035 if (params_.screenshare[video_idx].generate_slides) {
Artem Titov33f9d2b2019-12-05 14:59:001036 frame_generator = test::CreateSlideFrameGenerator(
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591037 kWidth, kHeight,
1038 params_.screenshare[video_idx].slide_change_interval *
1039 params_.video[video_idx].fps);
Johannes Kron525df592024-04-16 10:08:111040 } else if (!params_.video[video_idx].clip_path.empty()) {
1041 frame_generator = test::CreateFromYuvFileFrameGenerator(
1042 {params_.video[video_idx].clip_path}, params_.video[video_idx].width,
1043 params_.video[video_idx].height, 1);
ivica5d6a06c2015-09-17 12:30:241044 } else {
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591045 std::vector<std::string> slides = params_.screenshare[video_idx].slides;
Benjamin Wright1f4173e2019-03-14 00:59:321046 if (slides.empty()) {
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591047 slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv"));
1048 slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv"));
1049 slides.push_back(test::ResourcePath("photo_1850_1110", "yuv"));
1050 slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv"));
1051 }
1052 if (params_.screenshare[video_idx].scroll_duration == 0) {
1053 // Cycle image every slide_change_interval seconds.
Artem Titov33f9d2b2019-12-05 14:59:001054 frame_generator = test::CreateFromYuvFileFrameGenerator(
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591055 slides, kWidth, kHeight,
1056 params_.screenshare[video_idx].slide_change_interval *
1057 params_.video[video_idx].fps);
ivica5d6a06c2015-09-17 12:30:241058 } else {
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591059 RTC_CHECK_LE(params_.video[video_idx].width, kWidth);
1060 RTC_CHECK_LE(params_.video[video_idx].height, kHeight);
1061 RTC_CHECK_GT(params_.screenshare[video_idx].slide_change_interval, 0);
1062 const int kPauseDurationMs =
1063 (params_.screenshare[video_idx].slide_change_interval -
1064 params_.screenshare[video_idx].scroll_duration) *
1065 1000;
1066 RTC_CHECK_LE(params_.screenshare[video_idx].scroll_duration,
1067 params_.screenshare[video_idx].slide_change_interval);
1068
Artem Titov33f9d2b2019-12-05 14:59:001069 frame_generator = test::CreateScrollingInputFromYuvFilesFrameGenerator(
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591070 clock_, slides, kWidth, kHeight, params_.video[video_idx].width,
1071 params_.video[video_idx].height,
1072 params_.screenshare[video_idx].scroll_duration * 1000,
1073 kPauseDurationMs);
ivica5d6a06c2015-09-17 12:30:241074 }
1075 }
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591076 return frame_generator;
1077}
1078
1079void VideoQualityTest::CreateCapturers() {
Sebastian Jansson3bd2c792018-07-13 11:29:031080 RTC_DCHECK(video_sources_.empty());
Niels Möller1c931c42018-12-18 15:08:111081 video_sources_.resize(num_video_streams_);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591082 for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) {
Artem Titov33f9d2b2019-12-05 14:59:001083 std::unique_ptr<test::FrameGeneratorInterface> frame_generator;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591084 if (params_.screenshare[video_idx].enabled) {
Danil Chapovalov908e22e2019-04-18 12:34:161085 frame_generator = CreateFrameGenerator(video_idx);
1086 } else if (params_.video[video_idx].clip_path == "Generator") {
Artem Titov33f9d2b2019-12-05 14:59:001087 frame_generator = test::CreateSquareFrameGenerator(
Danil Chapovalov908e22e2019-04-18 12:34:161088 static_cast<int>(params_.video[video_idx].width),
1089 static_cast<int>(params_.video[video_idx].height), absl::nullopt,
1090 absl::nullopt);
1091 } else if (params_.video[video_idx].clip_path == "GeneratorI420A") {
Artem Titov33f9d2b2019-12-05 14:59:001092 frame_generator = test::CreateSquareFrameGenerator(
Danil Chapovalov908e22e2019-04-18 12:34:161093 static_cast<int>(params_.video[video_idx].width),
1094 static_cast<int>(params_.video[video_idx].height),
Artem Titov33f9d2b2019-12-05 14:59:001095 test::FrameGeneratorInterface::OutputType::kI420A, absl::nullopt);
Danil Chapovalov908e22e2019-04-18 12:34:161096 } else if (params_.video[video_idx].clip_path == "GeneratorI010") {
Artem Titov33f9d2b2019-12-05 14:59:001097 frame_generator = test::CreateSquareFrameGenerator(
Danil Chapovalov908e22e2019-04-18 12:34:161098 static_cast<int>(params_.video[video_idx].width),
1099 static_cast<int>(params_.video[video_idx].height),
Artem Titov33f9d2b2019-12-05 14:59:001100 test::FrameGeneratorInterface::OutputType::kI010, absl::nullopt);
Evan Shrubsole55c17862020-09-28 08:16:001101 } else if (params_.video[video_idx].clip_path == "GeneratorNV12") {
1102 frame_generator = test::CreateSquareFrameGenerator(
1103 static_cast<int>(params_.video[video_idx].width),
1104 static_cast<int>(params_.video[video_idx].height),
1105 test::FrameGeneratorInterface::OutputType::kNV12, absl::nullopt);
Danil Chapovalov908e22e2019-04-18 12:34:161106 } else if (params_.video[video_idx].clip_path.empty()) {
1107 video_sources_[video_idx] = test::CreateVideoCapturer(
1108 params_.video[video_idx].width, params_.video[video_idx].height,
1109 params_.video[video_idx].fps,
1110 params_.video[video_idx].capture_device_index);
1111 if (video_sources_[video_idx]) {
1112 continue;
1113 } else {
1114 // Failed to get actual camera, use chroma generator as backup.
Artem Titov33f9d2b2019-12-05 14:59:001115 frame_generator = test::CreateSquareFrameGenerator(
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591116 static_cast<int>(params_.video[video_idx].width),
Danil Chapovalovb9b146c2018-06-15 10:28:071117 static_cast<int>(params_.video[video_idx].height), absl::nullopt,
Danil Chapovalov908e22e2019-04-18 12:34:161118 absl::nullopt);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591119 }
Danil Chapovalov908e22e2019-04-18 12:34:161120 } else {
Artem Titov33f9d2b2019-12-05 14:59:001121 frame_generator = test::CreateFromYuvFileFrameGenerator(
Danil Chapovalov908e22e2019-04-18 12:34:161122 {params_.video[video_idx].clip_path}, params_.video[video_idx].width,
1123 params_.video[video_idx].height, 1);
1124 ASSERT_TRUE(frame_generator) << "Could not create capturer for "
1125 << params_.video[video_idx].clip_path
1126 << ".yuv. Is this file present?";
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591127 }
Danil Chapovalov908e22e2019-04-18 12:34:161128 ASSERT_TRUE(frame_generator);
1129 auto frame_generator_capturer =
Mirko Bonadei317a1f02019-09-17 15:06:181130 std::make_unique<test::FrameGeneratorCapturer>(
Danil Chapovalov908e22e2019-04-18 12:34:161131 clock_, std::move(frame_generator), params_.video[video_idx].fps,
1132 *task_queue_factory_);
1133 EXPECT_TRUE(frame_generator_capturer->Init());
1134 video_sources_[video_idx] = std::move(frame_generator_capturer);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591135 }
ivica5d6a06c2015-09-17 12:30:241136}
1137
Christoffer Rodbro39a44b22018-08-07 15:07:241138void VideoQualityTest::StartAudioStreams() {
1139 audio_send_stream_->Start();
Tommi3176ef72022-05-22 18:47:281140 for (AudioReceiveStreamInterface* audio_recv_stream : audio_receive_streams_)
Christoffer Rodbro39a44b22018-08-07 15:07:241141 audio_recv_stream->Start();
1142}
1143
Christoffer Rodbroc2a02882018-08-07 12:10:561144void VideoQualityTest::StartThumbnails() {
1145 for (VideoSendStream* send_stream : thumbnail_send_streams_)
1146 send_stream->Start();
Tommif6f45432022-05-20 13:21:201147 for (VideoReceiveStreamInterface* receive_stream : thumbnail_receive_streams_)
Christoffer Rodbroc2a02882018-08-07 12:10:561148 receive_stream->Start();
Christoffer Rodbroc2a02882018-08-07 12:10:561149}
1150
1151void VideoQualityTest::StopThumbnails() {
Tommif6f45432022-05-20 13:21:201152 for (VideoReceiveStreamInterface* receive_stream : thumbnail_receive_streams_)
Christoffer Rodbroc2a02882018-08-07 12:10:561153 receive_stream->Stop();
1154 for (VideoSendStream* send_stream : thumbnail_send_streams_)
1155 send_stream->Stop();
1156}
1157
Christoffer Rodbrob4bb4eb2017-11-13 12:03:521158std::unique_ptr<test::LayerFilteringTransport>
1159VideoQualityTest::CreateSendTransport() {
Artem Titov8ea1e9d2018-10-04 12:46:311160 std::unique_ptr<NetworkBehaviorInterface> network_behavior = nullptr;
Artem Titove269cb42018-08-29 07:59:231161 if (injection_components_->sender_network == nullptr) {
Mirko Bonadei317a1f02019-09-17 15:06:181162 network_behavior = std::make_unique<SimulatedNetwork>(*params_.config);
Artem Titove269cb42018-08-29 07:59:231163 } else {
Artem Titov8ea1e9d2018-10-04 12:46:311164 network_behavior = std::move(injection_components_->sender_network);
Artem Titove269cb42018-08-29 07:59:231165 }
Mirko Bonadei317a1f02019-09-17 15:06:181166 return std::make_unique<test::LayerFilteringTransport>(
Danil Chapovalovd15a0282019-10-22 08:48:171167 task_queue(),
Mirko Bonadei317a1f02019-09-17 15:06:181168 std::make_unique<FakeNetworkPipe>(clock_, std::move(network_behavior)),
Artem Titov8a9f3a82023-04-25 07:56:491169 sender_call_.get(), test::VideoTestConstants::kPayloadTypeVP8,
1170 test::VideoTestConstants::kPayloadTypeVP9, params_.video[0].selected_tl,
1171 params_.ss[0].selected_sl, payload_type_map_,
1172 test::VideoTestConstants::kVideoSendSsrcs[0],
1173 static_cast<uint32_t>(test::VideoTestConstants::kVideoSendSsrcs[0] +
1174 params_.ss[0].streams.size() - 1),
Per Kjellander89870ff2023-01-19 15:45:581175 GetRegisteredExtensions(), GetRegisteredExtensions());
Christoffer Rodbrob4bb4eb2017-11-13 12:03:521176}
1177
1178std::unique_ptr<test::DirectTransport>
1179VideoQualityTest::CreateReceiveTransport() {
Artem Titov8ea1e9d2018-10-04 12:46:311180 std::unique_ptr<NetworkBehaviorInterface> network_behavior = nullptr;
Artem Titove269cb42018-08-29 07:59:231181 if (injection_components_->receiver_network == nullptr) {
Mirko Bonadei317a1f02019-09-17 15:06:181182 network_behavior = std::make_unique<SimulatedNetwork>(*params_.config);
Artem Titove269cb42018-08-29 07:59:231183 } else {
Artem Titov8ea1e9d2018-10-04 12:46:311184 network_behavior = std::move(injection_components_->receiver_network);
Artem Titove269cb42018-08-29 07:59:231185 }
Mirko Bonadei317a1f02019-09-17 15:06:181186 return std::make_unique<test::DirectTransport>(
Danil Chapovalovd15a0282019-10-22 08:48:171187 task_queue(),
Mirko Bonadei317a1f02019-09-17 15:06:181188 std::make_unique<FakeNetworkPipe>(clock_, std::move(network_behavior)),
Per Kjellander89870ff2023-01-19 15:45:581189 receiver_call_.get(), payload_type_map_, GetRegisteredExtensions(),
1190 GetRegisteredExtensions());
Christoffer Rodbrob4bb4eb2017-11-13 12:03:521191}
1192
sprang7a975f72015-10-12 13:33:211193void VideoQualityTest::RunWithAnalyzer(const Params& params) {
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591194 num_video_streams_ = params.call.dual_video ? 2 : 1;
eladalon413ee9a2017-08-22 11:02:521195 std::unique_ptr<test::LayerFilteringTransport> send_transport;
1196 std::unique_ptr<test::DirectTransport> recv_transport;
1197 FILE* graph_data_output_file = nullptr;
eladalon413ee9a2017-08-22 11:02:521198
sprangce4aef12015-11-02 15:23:201199 params_ = params;
ivica5d6a06c2015-09-17 12:30:241200 // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to
1201 // differentiate between the analyzer and the renderer case.
Artem Titove269cb42018-08-29 07:59:231202 CheckParamsAndInjectionComponents();
ivica5d6a06c2015-09-17 12:30:241203
sprangce4aef12015-11-02 15:23:201204 if (!params_.analyzer.graph_data_output_filename.empty()) {
ivica5d6a06c2015-09-17 12:30:241205 graph_data_output_file =
sprangce4aef12015-11-02 15:23:201206 fopen(params_.analyzer.graph_data_output_filename.c_str(), "w");
Peter Boström74f6e9e2016-04-04 15:56:101207 RTC_CHECK(graph_data_output_file)
sprangce4aef12015-11-02 15:23:201208 << "Can't open the file " << params_.analyzer.graph_data_output_filename
1209 << "!";
ivica87f83a92015-10-08 12:13:321210 }
sprang7a975f72015-10-12 13:33:211211
ilnik98436952017-07-13 07:47:031212 if (!params.logging.rtc_event_log_name.empty()) {
Danil Chapovalovc03d8b62023-11-29 14:54:321213 std::unique_ptr<RtcEventLog> send_event_log =
1214 rtc_event_log_factory_.Create(env());
1215 std::unique_ptr<RtcEventLog> recv_event_log =
1216 rtc_event_log_factory_.Create(env());
Ilya Nikolaevskiya4259f62017-12-05 12:19:451217 std::unique_ptr<RtcEventLogOutputFile> send_output(
Mirko Bonadei317a1f02019-09-17 15:06:181218 std::make_unique<RtcEventLogOutputFile>(
Ilya Nikolaevskiya4259f62017-12-05 12:19:451219 params.logging.rtc_event_log_name + "_send",
1220 RtcEventLog::kUnlimitedOutput));
1221 std::unique_ptr<RtcEventLogOutputFile> recv_output(
Mirko Bonadei317a1f02019-09-17 15:06:181222 std::make_unique<RtcEventLogOutputFile>(
Ilya Nikolaevskiya4259f62017-12-05 12:19:451223 params.logging.rtc_event_log_name + "_recv",
1224 RtcEventLog::kUnlimitedOutput));
1225 bool event_log_started =
Danil Chapovalovc03d8b62023-11-29 14:54:321226 send_event_log->StartLogging(std::move(send_output),
1227 RtcEventLog::kImmediateOutput) &&
1228 recv_event_log->StartLogging(std::move(recv_output),
1229 RtcEventLog::kImmediateOutput);
ilnik98436952017-07-13 07:47:031230 RTC_DCHECK(event_log_started);
Danil Chapovalovc03d8b62023-11-29 14:54:321231 SetSendEventLog(std::move(send_event_log));
1232 SetRecvEventLog(std::move(recv_event_log));
ilnik98436952017-07-13 07:47:031233 }
1234
Danil Chapovalove519f382022-08-11 10:26:091235 SendTask(task_queue(), [this, &params, &send_transport, &recv_transport]() {
Danil Chapovalovc03d8b62023-11-29 14:54:321236 CallConfig send_call_config = SendCallConfig();
1237 CallConfig recv_call_config = RecvCallConfig();
Danil Chapovalove519f382022-08-11 10:26:091238 send_call_config.bitrate_config = params.call.call_bitrate_config;
1239 recv_call_config.bitrate_config = params.call.call_bitrate_config;
1240 if (params_.audio.enabled)
1241 InitializeAudioDevice(&send_call_config, &recv_call_config,
1242 params_.audio.use_real_adm);
Christoffer Rodbroc2a02882018-08-07 12:10:561243
Tony Herre5079e8a2024-07-29 05:54:201244 CreateCalls(std::move(send_call_config), std::move(recv_call_config));
Danil Chapovalove519f382022-08-11 10:26:091245 send_transport = CreateSendTransport();
1246 recv_transport = CreateReceiveTransport();
1247 });
stefanf116bd02015-10-27 15:29:421248
sprangce4aef12015-11-02 15:23:201249 std::string graph_title = params_.analyzer.graph_title;
1250 if (graph_title.empty())
1251 graph_title = VideoQualityTest::GenerateGraphTitle();
Danil Chapovalovf2443a72023-10-09 14:39:401252 bool is_quick_test_enabled = absl::GetFlag(FLAGS_webrtc_quick_perf_test);
Mirko Bonadei317a1f02019-09-17 15:06:181253 analyzer_ = std::make_unique<VideoAnalyzer>(
eladalon413ee9a2017-08-22 11:02:521254 send_transport.get(), params_.analyzer.test_label,
ilnik2a8c2f52017-02-15 10:23:281255 params_.analyzer.avg_psnr_threshold, params_.analyzer.avg_ssim_threshold,
ilnik9ae0d762017-02-15 08:53:121256 is_quick_test_enabled
1257 ? kFramesSentInQuickTest
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591258 : params_.analyzer.test_durations_secs * params_.video[0].fps,
Ilya Nikolaevskiy06c70952020-03-16 12:01:251259 is_quick_test_enabled
1260 ? TimeDelta::Millis(1)
1261 : TimeDelta::Seconds(params_.analyzer.test_durations_secs),
sprangce4aef12015-11-02 15:23:201262 graph_data_output_file, graph_title,
Artem Titov8a9f3a82023-04-25 07:56:491263 test::VideoTestConstants::kVideoSendSsrcs[params_.ss[0].selected_stream],
1264 test::VideoTestConstants::kSendRtxSsrcs[params_.ss[0].selected_stream],
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591265 static_cast<size_t>(params_.ss[0].selected_stream),
1266 params.ss[0].selected_sl, params_.video[0].selected_tl,
Artem Titovff7730d2019-04-02 11:46:531267 is_quick_test_enabled, clock_, params_.logging.rtp_dump_name,
Danil Chapovalovd15a0282019-10-22 08:48:171268 task_queue());
ivica5d6a06c2015-09-17 12:30:241269
Danil Chapovalove519f382022-08-11 10:26:091270 SendTask(task_queue(), [&]() {
Niels Möller88be9722018-10-10 08:58:521271 analyzer_->SetCall(sender_call_.get());
1272 analyzer_->SetReceiver(receiver_call_->Receiver());
1273 send_transport->SetReceiver(analyzer_.get());
eladalon413ee9a2017-08-22 11:02:521274 recv_transport->SetReceiver(sender_call_->Receiver());
ivica5d6a06c2015-09-17 12:30:241275
Niels Möller88be9722018-10-10 08:58:521276 SetupVideo(analyzer_.get(), recv_transport.get());
1277 SetupThumbnails(analyzer_.get(), recv_transport.get());
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591278 video_receive_configs_[params_.ss[0].selected_stream].renderer =
Niels Möller88be9722018-10-10 08:58:521279 analyzer_.get();
kthelgason2bc68642017-02-07 15:02:221280
eladalon413ee9a2017-08-22 11:02:521281 CreateFlexfecStreams();
1282 CreateVideoStreams();
Niels Möller88be9722018-10-10 08:58:521283 analyzer_->SetSendStream(video_send_streams_[0]);
Ilya Nikolaevskiyd47d3eb2019-01-21 15:27:171284 analyzer_->SetReceiveStream(
1285 video_receive_streams_[params_.ss[0].selected_stream]);
ivica5d6a06c2015-09-17 12:30:241286
Niels Möller88be9722018-10-10 08:58:521287 GetVideoSendStream()->SetSource(analyzer_->OutputInterface(),
Sebastian Jansson3bd2c792018-07-13 11:29:031288 degradation_preference_);
eladalon413ee9a2017-08-22 11:02:521289 SetupThumbnailCapturers(params_.call.num_thumbnails);
1290 for (size_t i = 0; i < thumbnail_send_streams_.size(); ++i) {
1291 thumbnail_send_streams_[i]->SetSource(thumbnail_capturers_[i].get(),
1292 degradation_preference_);
1293 }
ilnika014cc52017-03-07 12:21:041294
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591295 CreateCapturers();
ivicac1cc8542015-10-08 10:44:061296
Stefan Holmer1f7a0082019-01-11 14:39:081297 analyzer_->SetSource(video_sources_[0].get(), true);
ilnika014cc52017-03-07 12:21:041298
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591299 for (size_t video_idx = 1; video_idx < num_video_streams_; ++video_idx) {
Niels Möller1c931c42018-12-18 15:08:111300 video_send_streams_[video_idx]->SetSource(video_sources_[video_idx].get(),
1301 degradation_preference_);
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591302 }
1303
Christoffer Rodbroc2a02882018-08-07 12:10:561304 if (params_.audio.enabled) {
1305 SetupAudio(send_transport.get());
Christoffer Rodbro39a44b22018-08-07 15:07:241306 StartAudioStreams();
Niels Möller88be9722018-10-10 08:58:521307 analyzer_->SetAudioReceiveStream(audio_receive_streams_[0]);
Christoffer Rodbroc2a02882018-08-07 12:10:561308 }
Christoffer Rodbro39a44b22018-08-07 15:07:241309 StartVideoStreams();
1310 StartThumbnails();
Niels Möller88be9722018-10-10 08:58:521311 analyzer_->StartMeasuringCpuProcessTime();
eladalon413ee9a2017-08-22 11:02:521312 });
ivica5d6a06c2015-09-17 12:30:241313
Niels Möller88be9722018-10-10 08:58:521314 analyzer_->Wait();
ivica5d6a06c2015-09-17 12:30:241315
Danil Chapovalove519f382022-08-11 10:26:091316 SendTask(task_queue(), [&]() {
Christoffer Rodbroc2a02882018-08-07 12:10:561317 StopThumbnails();
1318 Stop();
ivica5d6a06c2015-09-17 12:30:241319
eladalon413ee9a2017-08-22 11:02:521320 DestroyStreams();
1321 DestroyThumbnailStreams();
ivica5d6a06c2015-09-17 12:30:241322
eladalon413ee9a2017-08-22 11:02:521323 if (graph_data_output_file)
1324 fclose(graph_data_output_file);
1325
eladalon413ee9a2017-08-22 11:02:521326 send_transport.reset();
1327 recv_transport.reset();
1328
1329 DestroyCalls();
1330 });
Niels Möller88be9722018-10-10 08:58:521331 analyzer_ = nullptr;
ivica5d6a06c2015-09-17 12:30:241332}
1333
henrika255750b2018-08-27 14:13:371334rtc::scoped_refptr<AudioDeviceModule> VideoQualityTest::CreateAudioDevice() {
1335#ifdef WEBRTC_WIN
Harald Alvestrand97597c02021-11-04 12:01:231336 RTC_LOG(LS_INFO) << "Using latest version of ADM on Windows";
Benjamin Wright1f4173e2019-03-14 00:59:321337 // We must initialize the COM library on a thread before we calling any of
1338 // the library functions. All COM functions in the ADM will return
1339 // CO_E_NOTINITIALIZED otherwise. The legacy ADM for Windows used internal
1340 // COM initialization but the new ADM requires COM to be initialized
1341 // externally.
Austin Orion0bb354c2020-10-29 18:30:101342 com_initializer_ =
1343 std::make_unique<ScopedCOMInitializer>(ScopedCOMInitializer::kMTA);
Benjamin Wright1f4173e2019-03-14 00:59:321344 RTC_CHECK(com_initializer_->Succeeded());
1345 RTC_CHECK(webrtc_win::core_audio_utility::IsSupported());
1346 RTC_CHECK(webrtc_win::core_audio_utility::IsMMCSSSupported());
Danil Chapovalov1c41be62019-04-01 07:16:121347 return CreateWindowsCoreAudioAudioDeviceModule(task_queue_factory_.get());
henrika255750b2018-08-27 14:13:371348#else
Benjamin Wright1f4173e2019-03-14 00:59:321349 // Use legacy factory method on all platforms except Windows.
Danil Chapovalov1c41be62019-04-01 07:16:121350 return AudioDeviceModule::Create(AudioDeviceModule::kPlatformDefaultAudio,
1351 task_queue_factory_.get());
henrika255750b2018-08-27 14:13:371352#endif
1353}
1354
Danil Chapovalova3ce4072023-10-13 11:53:001355void VideoQualityTest::InitializeAudioDevice(CallConfig* send_call_config,
1356 CallConfig* recv_call_config,
henrika255750b2018-08-27 14:13:371357 bool use_real_adm) {
1358 rtc::scoped_refptr<AudioDeviceModule> audio_device;
1359 if (use_real_adm) {
1360 // Run test with real ADM (using default audio devices) if user has
1361 // explicitly set the --audio and --use_real_adm command-line flags.
1362 audio_device = CreateAudioDevice();
1363 } else {
1364 // By default, create a test ADM which fakes audio.
Danil Chapovalov08fa9532019-06-12 11:49:171365 audio_device = TestAudioDeviceModule::Create(
1366 task_queue_factory_.get(),
henrika255750b2018-08-27 14:13:371367 TestAudioDeviceModule::CreatePulsedNoiseCapturer(32000, 48000),
1368 TestAudioDeviceModule::CreateDiscardRenderer(48000), 1.f);
1369 }
1370 RTC_CHECK(audio_device);
Christoffer Rodbroc2a02882018-08-07 12:10:561371
1372 AudioState::Config audio_state_config;
1373 audio_state_config.audio_mixer = AudioMixerImpl::Create();
1374 audio_state_config.audio_processing = AudioProcessingBuilder().Create();
henrika255750b2018-08-27 14:13:371375 audio_state_config.audio_device_module = audio_device;
Christoffer Rodbroc2a02882018-08-07 12:10:561376 send_call_config->audio_state = AudioState::Create(audio_state_config);
Christoffer Rodbroc2a02882018-08-07 12:10:561377 recv_call_config->audio_state = AudioState::Create(audio_state_config);
henrika255750b2018-08-27 14:13:371378 if (use_real_adm) {
1379 // The real ADM requires extra initialization: setting default devices,
1380 // setting up number of channels etc. Helper class also calls
1381 // AudioDeviceModule::Init().
1382 webrtc::adm_helpers::Init(audio_device.get());
1383 } else {
1384 audio_device->Init();
1385 }
1386 // Always initialize the ADM before injecting a valid audio transport.
1387 RTC_CHECK(audio_device->RegisterAudioCallback(
Benjamin Wright1f4173e2019-03-14 00:59:321388 send_call_config->audio_state->audio_transport()) == 0);
Christoffer Rodbroc2a02882018-08-07 12:10:561389}
1390
Sebastian Jansson3bd2c792018-07-13 11:29:031391void VideoQualityTest::SetupAudio(Transport* transport) {
Bjorn A Mellem7a9a0922019-11-26 17:19:401392 AudioSendStream::Config audio_send_config(transport);
Artem Titov8a9f3a82023-04-25 07:56:491393 audio_send_config.rtp.ssrc = test::VideoTestConstants::kAudioSendSsrc;
minyuea27172d2016-11-01 12:59:291394
1395 // Add extension to enable audio send side BWE, and allow audio bit rate
1396 // adaptation.
Sebastian Jansson3bd2c792018-07-13 11:29:031397 audio_send_config.rtp.extensions.clear();
1398 audio_send_config.send_codec_spec = AudioSendStream::Config::SendCodecSpec(
Artem Titov8a9f3a82023-04-25 07:56:491399 test::VideoTestConstants::kAudioSendPayloadType,
Yves Gerey665174f2018-06-19 13:03:051400 {"OPUS",
1401 48000,
1402 2,
1403 {{"usedtx", (params_.audio.dtx ? "1" : "0")}, {"stereo", "1"}}});
minyuea27172d2016-11-01 12:59:291404
Sebastian Jansson3bd2c792018-07-13 11:29:031405 if (params_.call.send_side_bwe) {
1406 audio_send_config.rtp.extensions.push_back(
1407 webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri,
Elad Alond8d32482019-02-18 22:45:571408 kTransportSequenceNumberExtensionId));
Sebastian Jansson3bd2c792018-07-13 11:29:031409 audio_send_config.min_bitrate_bps = kOpusMinBitrateBps;
1410 audio_send_config.max_bitrate_bps = kOpusBitrateFbBps;
1411 audio_send_config.send_codec_spec->transport_cc_enabled = true;
Minyue Li455d27c2019-01-07 12:14:301412 // Only allow ANA when send-side BWE is enabled.
1413 audio_send_config.audio_network_adaptor_config = params_.audio.ana_config;
Sebastian Jansson3bd2c792018-07-13 11:29:031414 }
1415 audio_send_config.encoder_factory = audio_encoder_factory_;
1416 SetAudioConfig(audio_send_config);
1417
Christoffer Rodbroc2a02882018-08-07 12:10:561418 std::string sync_group;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591419 if (params_.video[0].enabled && params_.audio.sync_video)
Sebastian Jansson3bd2c792018-07-13 11:29:031420 sync_group = kSyncGroup;
minyuea27172d2016-11-01 12:59:291421
Sebastian Jansson3bd2c792018-07-13 11:29:031422 CreateMatchingAudioConfigs(transport, sync_group);
1423 CreateAudioStreams();
minyuea27172d2016-11-01 12:59:291424}
1425
minyue73208662016-08-18 13:28:551426void VideoQualityTest::RunWithRenderers(const Params& params) {
Harald Alvestrand97597c02021-11-04 12:01:231427 RTC_LOG(LS_INFO) << __FUNCTION__;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591428 num_video_streams_ = params.call.dual_video ? 2 : 1;
eladalon413ee9a2017-08-22 11:02:521429 std::unique_ptr<test::LayerFilteringTransport> send_transport;
1430 std::unique_ptr<test::DirectTransport> recv_transport;
minyuea27172d2016-11-01 12:59:291431 std::unique_ptr<test::VideoRenderer> local_preview;
eladalon413ee9a2017-08-22 11:02:521432 std::vector<std::unique_ptr<test::VideoRenderer>> loopback_renderers;
Ilya Nikolaevskiy42e7d9c2018-11-02 12:14:371433
1434 if (!params.logging.rtc_event_log_name.empty()) {
Danil Chapovalovc03d8b62023-11-29 14:54:321435 std::unique_ptr<RtcEventLog> send_event_log =
1436 rtc_event_log_factory_.Create(env());
1437 std::unique_ptr<RtcEventLog> recv_event_log =
1438 rtc_event_log_factory_.Create(env());
Ilya Nikolaevskiy42e7d9c2018-11-02 12:14:371439 std::unique_ptr<RtcEventLogOutputFile> send_output(
Mirko Bonadei317a1f02019-09-17 15:06:181440 std::make_unique<RtcEventLogOutputFile>(
Ilya Nikolaevskiy42e7d9c2018-11-02 12:14:371441 params.logging.rtc_event_log_name + "_send",
1442 RtcEventLog::kUnlimitedOutput));
1443 std::unique_ptr<RtcEventLogOutputFile> recv_output(
Mirko Bonadei317a1f02019-09-17 15:06:181444 std::make_unique<RtcEventLogOutputFile>(
Ilya Nikolaevskiy42e7d9c2018-11-02 12:14:371445 params.logging.rtc_event_log_name + "_recv",
1446 RtcEventLog::kUnlimitedOutput));
1447 bool event_log_started =
Danil Chapovalovc03d8b62023-11-29 14:54:321448 send_event_log->StartLogging(std::move(send_output),
1449 /*output_period_ms=*/5000) &&
1450 recv_event_log->StartLogging(std::move(recv_output),
1451 /*output_period_ms=*/5000);
Ilya Nikolaevskiy42e7d9c2018-11-02 12:14:371452 RTC_DCHECK(event_log_started);
Danil Chapovalovc03d8b62023-11-29 14:54:321453 SetSendEventLog(std::move(send_event_log));
1454 SetRecvEventLog(std::move(recv_event_log));
Ilya Nikolaevskiy42e7d9c2018-11-02 12:14:371455 }
minyue73208662016-08-18 13:28:551456
Danil Chapovalove519f382022-08-11 10:26:091457 SendTask(task_queue(), [&]() {
eladalon413ee9a2017-08-22 11:02:521458 params_ = params;
Artem Titove269cb42018-08-29 07:59:231459 CheckParamsAndInjectionComponents();
palmkviste75f2042016-09-28 13:19:481460
Danil Chapovalova3ce4072023-10-13 11:53:001461 // TODO(ivica): Remove bitrate_config and use the default CallConfig(), to
eladalon413ee9a2017-08-22 11:02:521462 // match the full stack tests.
Danil Chapovalovc03d8b62023-11-29 14:54:321463 CallConfig send_call_config = SendCallConfig();
Christoffer Rodbro39a44b22018-08-07 15:07:241464 send_call_config.bitrate_config = params_.call.call_bitrate_config;
Danil Chapovalovc03d8b62023-11-29 14:54:321465 CallConfig recv_call_config = RecvCallConfig();
ivica5d6a06c2015-09-17 12:30:241466
Christoffer Rodbro39a44b22018-08-07 15:07:241467 if (params_.audio.enabled)
Benjamin Wright1f4173e2019-03-14 00:59:321468 InitializeAudioDevice(&send_call_config, &recv_call_config,
1469 params_.audio.use_real_adm);
minyue73208662016-08-18 13:28:551470
Tony Herre5079e8a2024-07-29 05:54:201471 CreateCalls(std::move(send_call_config), std::move(recv_call_config));
eladalon413ee9a2017-08-22 11:02:521472
1473 // TODO(minyue): consider if this is a good transport even for audio only
1474 // calls.
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591475 send_transport = CreateSendTransport();
eladalon413ee9a2017-08-22 11:02:521476
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591477 recv_transport = CreateReceiveTransport();
eladalon413ee9a2017-08-22 11:02:521478
1479 // TODO(ivica): Use two calls to be able to merge with RunWithAnalyzer or at
1480 // least share as much code as possible. That way this test would also match
1481 // the full stack tests better.
1482 send_transport->SetReceiver(receiver_call_->Receiver());
1483 recv_transport->SetReceiver(sender_call_->Receiver());
1484
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591485 if (params_.video[0].enabled) {
eladalon413ee9a2017-08-22 11:02:521486 // Create video renderers.
eladalon413ee9a2017-08-22 11:02:521487 SetupVideo(send_transport.get(), recv_transport.get());
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591488 size_t num_streams_processed = 0;
1489 for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) {
1490 const size_t selected_stream_id = params_.ss[video_idx].selected_stream;
1491 const size_t num_streams = params_.ss[video_idx].streams.size();
1492 if (selected_stream_id == num_streams) {
1493 for (size_t stream_id = 0; stream_id < num_streams; ++stream_id) {
Jonas Olsson366a50c2018-09-06 11:41:301494 rtc::StringBuilder oss;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591495 oss << "Loopback Video #" << video_idx << " - Stream #"
1496 << static_cast<int>(stream_id);
1497 loopback_renderers.emplace_back(test::VideoRenderer::Create(
1498 oss.str().c_str(),
1499 params_.ss[video_idx].streams[stream_id].width,
1500 params_.ss[video_idx].streams[stream_id].height));
1501 video_receive_configs_[stream_id + num_streams_processed].renderer =
1502 loopback_renderers.back().get();
1503 if (params_.audio.enabled && params_.audio.sync_video)
1504 video_receive_configs_[stream_id + num_streams_processed]
1505 .sync_group = kSyncGroup;
1506 }
1507 } else {
Jonas Olsson366a50c2018-09-06 11:41:301508 rtc::StringBuilder oss;
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591509 oss << "Loopback Video #" << video_idx;
1510 loopback_renderers.emplace_back(test::VideoRenderer::Create(
1511 oss.str().c_str(),
1512 params_.ss[video_idx].streams[selected_stream_id].width,
1513 params_.ss[video_idx].streams[selected_stream_id].height));
1514 video_receive_configs_[selected_stream_id + num_streams_processed]
1515 .renderer = loopback_renderers.back().get();
eladalon413ee9a2017-08-22 11:02:521516 if (params_.audio.enabled && params_.audio.sync_video)
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591517 video_receive_configs_[num_streams_processed + selected_stream_id]
1518 .sync_group = kSyncGroup;
eladalon413ee9a2017-08-22 11:02:521519 }
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591520 num_streams_processed += num_streams;
eladalon413ee9a2017-08-22 11:02:521521 }
eladalon413ee9a2017-08-22 11:02:521522 CreateFlexfecStreams();
1523 CreateVideoStreams();
1524
Ilya Nikolaevskiy255d1cd2017-12-21 17:02:591525 CreateCapturers();
Niels Möller412d1852019-01-02 08:42:541526 if (params_.video[0].enabled) {
1527 // Create local preview
1528 local_preview.reset(test::VideoRenderer::Create(
1529 "Local Preview", params_.video[0].width, params_.video[0].height));
1530
1531 video_sources_[0]->AddOrUpdateSink(local_preview.get(),
1532 rtc::VideoSinkWants());
1533 }
Sebastian Jansson3bd2c792018-07-13 11:29:031534 ConnectVideoSourcesToStreams();
eladalon413ee9a2017-08-22 11:02:521535 }
1536
1537 if (params_.audio.enabled) {
Sebastian Jansson3bd2c792018-07-13 11:29:031538 SetupAudio(send_transport.get());
eladalon413ee9a2017-08-22 11:02:521539 }
1540
Sebastian Jansson3bd2c792018-07-13 11:29:031541 Start();
eladalon413ee9a2017-08-22 11:02:521542 });
minyue73208662016-08-18 13:28:551543
Tommi9e46cf52020-05-04 14:43:051544 PressEnterToContinue(task_queue());
ivica5d6a06c2015-09-17 12:30:241545
Danil Chapovalove519f382022-08-11 10:26:091546 SendTask(task_queue(), [&]() {
Sebastian Jansson3bd2c792018-07-13 11:29:031547 Stop();
1548 DestroyStreams();
minyue73208662016-08-18 13:28:551549
eladalon413ee9a2017-08-22 11:02:521550 send_transport.reset();
1551 recv_transport.reset();
sprang1168fd42017-06-21 16:00:171552
eladalon413ee9a2017-08-22 11:02:521553 local_preview.reset();
1554 loopback_renderers.clear();
1555
1556 DestroyCalls();
1557 });
ivica5d6a06c2015-09-17 12:30:241558}
1559
ivica5d6a06c2015-09-17 12:30:241560} // namespace webrtc