blob: a77c38626531b3d746d6de45e98a93c61ff82dfa [file] [log] [blame]
perkj26091b12016-09-01 08:17:401/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 08:30:3111#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 15:24:5913#include <algorithm>
perkj803d97f2016-11-01 18:45:4614#include <limits>
Danil Chapovalovd3ba2362019-04-10 15:01:2315#include <memory>
Henrik Boström56db9ff2021-03-24 08:06:4516#include <tuple>
Per512ecb32016-09-23 13:52:0617#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 14:30:1619#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 15:07:0120#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 15:01:2321#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 12:01:3322#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 11:00:5523#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 09:20:0924#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 10:30:0025#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 08:06:4526#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 10:37:1627#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 18:02:5628#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3129#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 07:15:1330#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 09:36:5531#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 16:10:5732#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 08:06:4533#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 14:44:3034#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 12:57:5735#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 08:30:3136#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 10:56:2037#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 11:02:3638#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 15:26:0139#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 14:30:1640#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 13:59:1241#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 17:11:0042#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 15:50:0043#include "media/engine/webrtc_video_engine.h"
philipel95701502022-01-18 17:47:5244#include "modules/video_coding/codecs/av1/libaom_av1_encoder_supported.h"
Henrik Boström56db9ff2021-03-24 08:06:4545#include "modules/video_coding/codecs/h264/include/h264.h"
46#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
47#include "modules/video_coding/codecs/vp8/include/vp8.h"
48#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 17:32:3749#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 12:02:1350#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 10:24:3351#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 11:06:5952#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2353#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 09:39:5154#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 14:29:0955#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3156#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 17:11:0057#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 11:13:3258#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 06:51:1059#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3160#include "test/encoder_settings.h"
61#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 14:59:0062#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3163#include "test/gmock.h"
64#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 08:06:4565#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 14:12:0766#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2367#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 07:07:2468#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 11:00:5569#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3170#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 08:17:4071
72namespace webrtc {
73
sprang57c2fff2017-01-16 14:24:0274using ::testing::_;
philipeld9cc8c02019-09-16 12:53:4075using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 12:08:3976using ::testing::Eq;
philipeld9cc8c02019-09-16 12:53:4077using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 12:08:3978using ::testing::Ge;
79using ::testing::Gt;
Markus Handellee225432021-11-29 11:35:1280using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 12:08:3981using ::testing::Le;
82using ::testing::Lt;
philipel9b058032020-02-10 10:30:0083using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 11:00:5584using ::testing::Mock;
philipel9b058032020-02-10 10:30:0085using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 11:00:5586using ::testing::Optional;
philipel9b058032020-02-10 10:30:0087using ::testing::Return;
Per Kjellander4190ce92020-12-15 16:24:5588using ::testing::SizeIs;
philipeld9cc8c02019-09-16 12:53:4089using ::testing::StrictMock;
kthelgason876222f2016-11-29 09:44:1190
perkj803d97f2016-11-01 18:45:4691namespace {
Åsa Persson8c1bf952018-09-13 08:42:1992const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 14:56:0093const int kQpLow = 1;
94const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 08:42:1995const int kMinFramerateFps = 2;
96const int kMinBalancedFramerateFps = 7;
97const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 12:37:0098const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 08:07:1199const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
100const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
101const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
102const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 15:02:22103const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 11:21:07104const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 08:42:19105const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 08:48:48106const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 11:12:19107const VideoEncoder::ResolutionBitrateLimits
108 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
109const VideoEncoder::ResolutionBitrateLimits
110 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 12:37:00111
Asa Persson606d3cb2021-10-04 08:07:11112uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 14:30:16113 0x00, 0x00, 0x03, 0x03, 0xF4,
114 0x05, 0x03, 0xC7, 0xE0, 0x1B,
115 0x41, 0x10, 0x8D, 0x00};
116
Sergey Silkin0e42cf72021-03-15 09:12:57117const uint8_t kCodedFrameVp8Qp25[] = {
118 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
119 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
120 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
121
Markus Handell818e7fb2021-12-30 12:01:33122VideoFrame CreateSimpleNV12Frame() {
123 return VideoFrame::Builder()
124 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
125 /*width=*/16, /*height=*/16))
126 .build();
127}
128
Markus Handell8d87c462021-12-16 10:37:16129void PassAFrame(
130 TaskQueueBase* encoder_queue,
131 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
132 int64_t ntp_time_ms) {
133 encoder_queue->PostTask(
134 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
Markus Handell818e7fb2021-12-30 12:01:33135 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms),
136 1, CreateSimpleNV12Frame());
Markus Handell8d87c462021-12-16 10:37:16137 }));
138}
139
perkj803d97f2016-11-01 18:45:46140class TestBuffer : public webrtc::I420Buffer {
141 public:
142 TestBuffer(rtc::Event* event, int width, int height)
143 : I420Buffer(width, height), event_(event) {}
144
145 private:
146 friend class rtc::RefCountedObject<TestBuffer>;
147 ~TestBuffer() override {
148 if (event_)
149 event_->Set();
150 }
151 rtc::Event* const event_;
152};
153
Henrik Boström56db9ff2021-03-24 08:06:45154// A fake native buffer that can't be converted to I420. Upon scaling, it
155// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 13:59:12156class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
157 public:
158 FakeNativeBuffer(rtc::Event* event, int width, int height)
159 : event_(event), width_(width), height_(height) {}
160 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
161 int width() const override { return width_; }
162 int height() const override { return height_; }
163 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
164 return nullptr;
165 }
Henrik Boström56db9ff2021-03-24 08:06:45166 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
167 int offset_x,
168 int offset_y,
169 int crop_width,
170 int crop_height,
171 int scaled_width,
172 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 17:21:43173 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
174 scaled_height);
Henrik Boström56db9ff2021-03-24 08:06:45175 }
Noah Richards51db4212019-06-12 13:59:12176
177 private:
178 friend class rtc::RefCountedObject<FakeNativeBuffer>;
179 ~FakeNativeBuffer() override {
180 if (event_)
181 event_->Set();
182 }
183 rtc::Event* const event_;
184 const int width_;
185 const int height_;
186};
187
Evan Shrubsole895556e2020-10-05 07:15:13188// A fake native buffer that is backed by an NV12 buffer.
189class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
190 public:
191 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
192 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
193
194 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
195 int width() const override { return nv12_buffer_->width(); }
196 int height() const override { return nv12_buffer_->height(); }
197 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
198 return nv12_buffer_->ToI420();
199 }
Evan Shrubsoleb556b082020-10-08 12:56:45200 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
201 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
202 if (absl::c_find(types, Type::kNV12) != types.end()) {
203 return nv12_buffer_;
204 }
205 return nullptr;
206 }
Evan Shrubsole895556e2020-10-05 07:15:13207 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
208
209 private:
210 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
211 ~FakeNV12NativeBuffer() override {
212 if (event_)
213 event_->Set();
214 }
215 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
216 rtc::Event* const event_;
217};
218
Niels Möller7dc26b72017-12-06 09:27:48219class CpuOveruseDetectorProxy : public OveruseFrameDetector {
220 public:
Niels Möllerd1f7eb62018-03-28 14:40:58221 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
222 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 16:49:07223 last_target_framerate_fps_(-1),
224 framerate_updated_event_(true /* manual_reset */,
225 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 09:27:48226 virtual ~CpuOveruseDetectorProxy() {}
227
228 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 11:13:32229 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48230 last_target_framerate_fps_ = framerate_fps;
231 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 16:49:07232 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 09:27:48233 }
234
235 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 11:13:32236 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48237 return last_target_framerate_fps_;
238 }
239
Niels Möller4db138e2018-04-19 07:04:13240 CpuOveruseOptions GetOptions() { return options_; }
241
Henrik Boström381d1092020-05-12 16:49:07242 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
243
Niels Möller7dc26b72017-12-06 09:27:48244 private:
Markus Handella3765182020-07-08 11:13:32245 Mutex lock_;
Niels Möller7dc26b72017-12-06 09:27:48246 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 16:49:07247 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 09:27:48248};
249
Henrik Boström0f0aa9c2020-06-02 11:02:36250class FakeVideoSourceRestrictionsListener
251 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 16:49:07252 public:
Henrik Boström0f0aa9c2020-06-02 11:02:36253 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 16:49:07254 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 11:02:36255 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 16:49:07256 RTC_DCHECK(was_restrictions_updated_);
257 }
258
259 rtc::Event* restrictions_updated_event() {
260 return &restrictions_updated_event_;
261 }
262
Henrik Boström0f0aa9c2020-06-02 11:02:36263 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 16:49:07264 void OnVideoSourceRestrictionsUpdated(
265 VideoSourceRestrictions restrictions,
266 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 09:47:46267 rtc::scoped_refptr<Resource> reason,
268 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 16:49:07269 was_restrictions_updated_ = true;
270 restrictions_updated_event_.Set();
271 }
272
273 private:
274 bool was_restrictions_updated_;
275 rtc::Event restrictions_updated_event_;
276};
277
Evan Shrubsole5fd40602020-05-25 14:19:54278auto WantsFps(Matcher<int> fps_matcher) {
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_matcher);
281}
282
283auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
284 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
285 AllOf(max_pixel_matcher, Gt(0)));
286}
287
Evan Shrubsole5cd7eb82020-05-25 12:08:39288auto ResolutionMax() {
289 return AllOf(
Evan Shrubsole5fd40602020-05-25 14:19:54290 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 12:08:39291 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
292 Eq(absl::nullopt)));
293}
294
295auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 14:19:54296 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 12:08:39297}
298
299auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 14:19:54300 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 12:08:39301}
302
303auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54304 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 12:08:39305}
306
307auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54308 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 12:08:39309}
310
311auto FpsMaxResolutionMax() {
312 return AllOf(FpsMax(), ResolutionMax());
313}
314
315auto UnlimitedSinkWants() {
316 return AllOf(FpsUnlimited(), ResolutionMax());
317}
318
319auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
320 Matcher<int> fps_range_matcher;
321
322 if (last_frame_pixels <= 320 * 240) {
323 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 15:50:00324 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 12:08:39325 fps_range_matcher = AllOf(Ge(10), Le(15));
326 } else if (last_frame_pixels <= 640 * 480) {
327 fps_range_matcher = Ge(15);
328 } else {
329 fps_range_matcher = Eq(kDefaultFramerate);
330 }
331 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
332 fps_range_matcher);
333}
334
Evan Shrubsole5fd40602020-05-25 14:19:54335auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
336 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
337 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
338}
339
340auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
341 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
342}
343
344auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
345 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
346}
347
348auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
349 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
350 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
351}
352
353auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
354 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
355 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
356}
357
358auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
359 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
360 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
361}
362
363auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
364 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
365 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
366}
367
mflodmancc3d4422017-08-03 15:27:51368class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 18:45:46369 public:
Markus Handell9a478b52021-11-18 15:07:01370 VideoStreamEncoderUnderTest(
371 TimeController* time_controller,
372 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
373 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
374 encoder_queue,
375 SendStatisticsProxy* stats_proxy,
376 const VideoStreamEncoderSettings& settings,
377 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 14:12:07378 allocation_callback_type,
Erik Språnge4589cb2022-04-06 14:44:30379 const FieldTrialsView& field_trials,
380 int num_cores)
Markus Handell9a478b52021-11-18 15:07:01381 : VideoStreamEncoder(time_controller->GetClock(),
Erik Språnge4589cb2022-04-06 14:44:30382 num_cores,
Markus Handell9a478b52021-11-18 15:07:01383 stats_proxy,
384 settings,
385 std::unique_ptr<OveruseFrameDetector>(
386 overuse_detector_proxy_ =
387 new CpuOveruseDetectorProxy(stats_proxy)),
388 std::move(cadence_adapter),
389 std::move(encoder_queue),
Jonas Orelandc7f691a2022-03-09 14:12:07390 allocation_callback_type,
391 field_trials),
Tomas Gunnarsson612445e2020-09-21 12:31:23392 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 15:59:05393 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 11:02:36394 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 09:47:10395 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 14:29:22396 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 09:36:55397 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 14:29:22398 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 11:02:36399 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 15:26:01400 }
perkj803d97f2016-11-01 18:45:46401
Henrik Boström381d1092020-05-12 16:49:07402 void SetSourceAndWaitForRestrictionsUpdated(
403 rtc::VideoSourceInterface<VideoFrame>* source,
404 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 11:02:36405 FakeVideoSourceRestrictionsListener listener;
406 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07407 SetSource(source, degradation_preference);
408 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 11:02:36409 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07410 }
411
412 void SetSourceAndWaitForFramerateUpdated(
413 rtc::VideoSourceInterface<VideoFrame>* source,
414 const DegradationPreference& degradation_preference) {
415 overuse_detector_proxy_->framerate_updated_event()->Reset();
416 SetSource(source, degradation_preference);
417 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
418 }
419
420 void OnBitrateUpdatedAndWaitForManagedResources(
421 DataRate target_bitrate,
422 DataRate stable_target_bitrate,
423 DataRate link_allocation,
424 uint8_t fraction_lost,
425 int64_t round_trip_time_ms,
426 double cwnd_reduce_ratio) {
427 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
428 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
429 // Bitrate is updated on the encoder queue.
430 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 16:49:07431 }
432
kthelgason2fc52542017-03-03 08:24:41433 // This is used as a synchronisation mechanism, to make sure that the
434 // encoder queue is not blocked before we start sending it frames.
435 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 09:11:55436 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 08:24:41437 }
438
Henrik Boström91aa7322020-04-28 10:24:33439 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 13:25:32440 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 10:24:33441 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12442 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36443 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33444 event.Set();
445 });
446 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55447 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 10:24:33448 }
Tomas Gunnarsson612445e2020-09-21 12:31:23449
Henrik Boström91aa7322020-04-28 10:24:33450 void TriggerCpuUnderuse() {
451 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12452 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36453 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33454 event.Set();
455 });
456 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55457 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 13:25:32458 }
kthelgason876222f2016-11-29 09:44:11459
Henrik Boström91aa7322020-04-28 10:24:33460 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 13:25:32461 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 10:24:33462 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12463 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36464 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33465 event.Set();
466 });
467 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55468 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 13:25:32469 }
Åsa Perssonb67c44c2019-09-24 13:25:32470 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 10:24:33471 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12472 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36473 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33474 event.Set();
475 });
476 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55477 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 10:24:33478 }
479
Tomas Gunnarsson612445e2020-09-21 12:31:23480 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 09:27:48481 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 14:29:22482 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
483 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 11:02:36484 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 18:45:46485};
486
Noah Richards51db4212019-06-12 13:59:12487// Simulates simulcast behavior and makes highest stream resolutions divisible
488// by 4.
489class CroppingVideoStreamFactory
490 : public VideoEncoderConfig::VideoStreamFactoryInterface {
491 public:
Åsa Persson17b29b92020-10-17 10:57:58492 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 13:59:12493
494 private:
495 std::vector<VideoStream> CreateEncoderStreams(
496 int width,
497 int height,
498 const VideoEncoderConfig& encoder_config) override {
499 std::vector<VideoStream> streams = test::CreateVideoStreams(
500 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 13:59:12501 return streams;
502 }
Noah Richards51db4212019-06-12 13:59:12503};
504
sprangb1ca0732017-02-01 16:38:12505class AdaptingFrameForwarder : public test::FrameForwarder {
506 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23507 explicit AdaptingFrameForwarder(TimeController* time_controller)
508 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 12:51:49509 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 16:38:12510
511 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 11:13:32512 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12513 adaptation_enabled_ = enabled;
514 }
515
asaperssonfab67072017-04-04 12:51:49516 bool adaption_enabled() const {
Markus Handella3765182020-07-08 11:13:32517 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12518 return adaptation_enabled_;
519 }
520
Henrik Boström1124ed12021-02-25 09:30:39521 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
522 // the resolution or frame rate was different than it is currently. If
523 // something else is modified, such as encoder resolutions, but the resolution
524 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-16 06:40:18525 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 11:13:32526 MutexLock lock(&mutex_);
asapersson09f05612017-05-16 06:40:18527 return last_wants_;
528 }
529
Danil Chapovalovb9b146c2018-06-15 10:28:07530 absl::optional<int> last_sent_width() const { return last_width_; }
531 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-09 01:04:29532
sprangb1ca0732017-02-01 16:38:12533 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 12:31:23534 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 09:11:55535 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:23536
sprangb1ca0732017-02-01 16:38:12537 int cropped_width = 0;
538 int cropped_height = 0;
539 int out_width = 0;
540 int out_height = 0;
sprangc5d62e22017-04-03 06:53:04541 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23542 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
543 << "w=" << video_frame.width()
544 << "h=" << video_frame.height();
sprangc5d62e22017-04-03 06:53:04545 if (adapter_.AdaptFrameResolution(
546 video_frame.width(), video_frame.height(),
547 video_frame.timestamp_us() * 1000, &cropped_width,
548 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 22:49:37549 VideoFrame adapted_frame =
550 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43551 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 22:49:37552 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 17:05:30553 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 22:49:37554 .set_timestamp_ms(99)
555 .set_rotation(kVideoRotation_0)
556 .build();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:17557 if (video_frame.has_update_rect()) {
558 adapted_frame.set_update_rect(
559 video_frame.update_rect().ScaleWithFrame(
560 video_frame.width(), video_frame.height(), 0, 0,
561 video_frame.width(), video_frame.height(), out_width,
562 out_height));
563 }
sprangc5d62e22017-04-03 06:53:04564 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-09 01:04:29565 last_width_.emplace(adapted_frame.width());
566 last_height_.emplace(adapted_frame.height());
567 } else {
Danil Chapovalovb9b146c2018-06-15 10:28:07568 last_width_ = absl::nullopt;
569 last_height_ = absl::nullopt;
sprangc5d62e22017-04-03 06:53:04570 }
sprangb1ca0732017-02-01 16:38:12571 } else {
Harald Alvestrand97597c02021-11-04 12:01:23572 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 16:38:12573 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-09 01:04:29574 last_width_.emplace(video_frame.width());
575 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 16:38:12576 }
577 }
578
Åsa Perssonf5f7e8e2021-06-09 20:55:38579 void OnOutputFormatRequest(int width, int height) {
580 absl::optional<std::pair<int, int>> target_aspect_ratio =
581 std::make_pair(width, height);
582 absl::optional<int> max_pixel_count = width * height;
583 absl::optional<int> max_fps;
584 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
585 max_fps);
586 }
587
sprangb1ca0732017-02-01 16:38:12588 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
589 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 11:13:32590 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 09:30:39591 rtc::VideoSinkWants prev_wants = sink_wants_locked();
592 bool did_adapt =
593 prev_wants.max_pixel_count != wants.max_pixel_count ||
594 prev_wants.target_pixel_count != wants.target_pixel_count ||
595 prev_wants.max_framerate_fps != wants.max_framerate_fps;
596 if (did_adapt) {
597 last_wants_ = prev_wants;
598 }
Rasmus Brandt287e4642019-11-15 15:56:01599 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 06:37:30600 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 16:38:12601 }
Tomas Gunnarsson612445e2020-09-21 12:31:23602
603 TimeController* const time_controller_;
sprangb1ca0732017-02-01 16:38:12604 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 11:13:32605 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
606 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 10:28:07607 absl::optional<int> last_width_;
608 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 16:38:12609};
sprangc5d62e22017-04-03 06:53:04610
Niels Möller213618e2018-07-24 07:29:58611// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-03 06:53:04612class MockableSendStatisticsProxy : public SendStatisticsProxy {
613 public:
614 MockableSendStatisticsProxy(Clock* clock,
615 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 11:52:48616 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 09:04:48617 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 11:52:48618 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-03 06:53:04619
620 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 11:13:32621 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04622 if (mock_stats_)
623 return *mock_stats_;
624 return SendStatisticsProxy::GetStats();
625 }
626
Niels Möller213618e2018-07-24 07:29:58627 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 11:13:32628 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 07:29:58629 if (mock_stats_)
630 return mock_stats_->input_frame_rate;
631 return SendStatisticsProxy::GetInputFrameRate();
632 }
sprangc5d62e22017-04-03 06:53:04633 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 11:13:32634 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04635 mock_stats_.emplace(stats);
636 }
637
638 void ResetMockStats() {
Markus Handella3765182020-07-08 11:13:32639 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04640 mock_stats_.reset();
641 }
642
Tomas Gunnarsson612445e2020-09-21 12:31:23643 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
644 on_frame_dropped_ = std::move(callback);
645 }
646
sprangc5d62e22017-04-03 06:53:04647 private:
Tomas Gunnarsson612445e2020-09-21 12:31:23648 void OnFrameDropped(DropReason reason) override {
649 SendStatisticsProxy::OnFrameDropped(reason);
650 if (on_frame_dropped_)
651 on_frame_dropped_(reason);
652 }
653
Markus Handella3765182020-07-08 11:13:32654 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 10:28:07655 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 12:31:23656 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-03 06:53:04657};
658
Markus Handellb4e96d42021-11-05 11:00:55659class SimpleVideoStreamEncoderFactory {
660 public:
661 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
662 public:
663 using VideoStreamEncoder::VideoStreamEncoder;
664 ~AdaptedVideoStreamEncoder() { Stop(); }
665 };
666
Markus Handell8d87c462021-12-16 10:37:16667 class MockFakeEncoder : public test::FakeEncoder {
668 public:
669 using FakeEncoder::FakeEncoder;
670 MOCK_METHOD(CodecSpecificInfo,
671 EncodeHook,
672 (EncodedImage & encoded_image,
673 rtc::scoped_refptr<EncodedImageBuffer> buffer),
674 (override));
675 };
676
Markus Handellee225432021-11-29 11:35:12677 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 11:00:55678 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 11:35:12679 encoder_settings_.bitrate_allocator_factory =
680 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 11:00:55681 }
682
Markus Handell818e7fb2021-12-30 12:01:33683 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 11:35:12684 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 14:12:07685 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 09:04:48686 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 11:00:55687 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
688 time_controller_.GetClock(),
689 /*number_of_cores=*/1,
690 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
691 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 11:35:12692 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 11:00:55693 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 14:12:07694 kVideoBitrateAllocation,
695 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 11:00:55696 result->SetSink(&sink_, /*rotation_applied=*/false);
697 return result;
698 }
699
Markus Handell818e7fb2021-12-30 12:01:33700 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
701 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
702 TaskQueueBase** encoder_queue_ptr = nullptr) {
703 auto encoder_queue =
704 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
705 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
706 if (encoder_queue_ptr)
707 *encoder_queue_ptr = encoder_queue.get();
708 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
709 std::move(encoder_queue));
710 }
711
Markus Handell9a478b52021-11-18 15:07:01712 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 10:37:16713 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 15:07:01714
Markus Handell818e7fb2021-12-30 12:01:33715 GlobalSimulatedTimeController* GetTimeController() {
716 return &time_controller_;
717 }
718
Markus Handellb4e96d42021-11-05 11:00:55719 private:
720 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
721 public:
722 ~NullEncoderSink() override = default;
723 void OnEncoderConfigurationChanged(
724 std::vector<VideoStream> streams,
725 bool is_svc,
726 VideoEncoderConfig::ContentType content_type,
727 int min_transmit_bitrate_bps) override {}
728 void OnBitrateAllocationUpdated(
729 const VideoBitrateAllocation& allocation) override {}
730 void OnVideoLayersAllocationUpdated(
731 VideoLayersAllocation allocation) override {}
732 Result OnEncodedImage(
733 const EncodedImage& encoded_image,
734 const CodecSpecificInfo* codec_specific_info) override {
735 return Result(EncodedImageCallback::Result::OK);
736 }
737 };
738
Jonas Orelandc7f691a2022-03-09 14:12:07739 test::ScopedKeyValueConfig field_trials_;
Markus Handellee225432021-11-29 11:35:12740 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
741 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
742 time_controller_.CreateTaskQueueFactory()};
743 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
744 std::make_unique<MockableSendStatisticsProxy>(
745 time_controller_.GetClock(),
746 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 11:52:48747 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
748 field_trials_);
Markus Handellee225432021-11-29 11:35:12749 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
750 CreateBuiltinVideoBitrateAllocatorFactory();
751 VideoStreamEncoderSettings encoder_settings_{
752 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 10:37:16753 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
754 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 11:00:55755 NullEncoderSink sink_;
756};
757
758class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
759 public:
760 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 10:37:16761 MOCK_METHOD(void,
762 SetZeroHertzModeEnabled,
763 (absl::optional<ZeroHertzModeParams>),
764 (override));
Markus Handellb4e96d42021-11-05 11:00:55765 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 11:35:12766 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
767 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 10:37:16768 MOCK_METHOD(void,
769 UpdateLayerQualityConvergence,
770 (int spatial_index, bool converged),
771 (override));
772 MOCK_METHOD(void,
773 UpdateLayerStatus,
774 (int spatial_index, bool enabled),
775 (override));
Markus Handell818e7fb2021-12-30 12:01:33776 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 11:00:55777};
778
philipel9b058032020-02-10 10:30:00779class MockEncoderSelector
780 : public VideoEncoderFactory::EncoderSelectorInterface {
781 public:
Danil Chapovalov91fdc602020-05-14 17:17:51782 MOCK_METHOD(void,
783 OnCurrentEncoder,
784 (const SdpVideoFormat& format),
785 (override));
786 MOCK_METHOD(absl::optional<SdpVideoFormat>,
787 OnAvailableBitrate,
788 (const DataRate& rate),
789 (override));
790 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 10:30:00791};
792
Markus Handell2e0f4f02021-12-21 18:14:58793class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
794 public:
795 MOCK_METHOD(void,
796 AddOrUpdateSink,
797 (rtc::VideoSinkInterface<VideoFrame>*,
798 const rtc::VideoSinkWants&),
799 (override));
800 MOCK_METHOD(void,
801 RemoveSink,
802 (rtc::VideoSinkInterface<VideoFrame>*),
803 (override));
804 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
805};
806
perkj803d97f2016-11-01 18:45:46807} // namespace
808
mflodmancc3d4422017-08-03 15:27:51809class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 08:17:40810 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23811 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 08:17:40812
mflodmancc3d4422017-08-03 15:27:51813 VideoStreamEncoderTest()
perkj26091b12016-09-01 08:17:40814 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-03 06:45:26815 codec_width_(320),
816 codec_height_(240),
Åsa Persson8c1bf952018-09-13 08:42:19817 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 12:31:23818 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 07:04:13819 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-03 06:53:04820 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 12:31:23821 time_controller_.GetClock(),
perkj803d97f2016-11-01 18:45:46822 video_send_config_,
Jonas Oreland8ca06132022-03-14 11:52:48823 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
824 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 12:31:23825 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 08:17:40826
827 void SetUp() override {
perkj803d97f2016-11-01 18:45:46828 metrics::Reset();
perkj26091b12016-09-01 08:17:40829 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 07:04:13830 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56831 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 12:18:34832 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 13:36:51833 video_send_config_.rtp.payload_name = "FAKE";
834 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 08:17:40835
Per512ecb32016-09-23 13:52:06836 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51837 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 06:57:51838 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
839 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
840 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 15:41:30841 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 14:06:52842
Niels Möllerf1338562018-04-26 07:51:47843 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 12:37:00844 }
845
Per Kjellanderb03b6c82021-01-03 09:26:03846 void ConfigureEncoder(
847 VideoEncoderConfig video_encoder_config,
848 VideoStreamEncoder::BitrateAllocationCallbackType
849 allocation_callback_type =
850 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 14:44:30851 kVideoBitrateAllocationWhenScreenSharing,
852 int num_cores = 1) {
mflodmancc3d4422017-08-03 15:27:51853 if (video_stream_encoder_)
854 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 15:07:01855
856 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
857 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
858 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
859 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
860 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 11:52:48861 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 15:07:01862 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
863 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
864 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 14:44:30865 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 08:07:11866 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 15:27:51867 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:41868 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 08:07:11869 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 15:27:51870 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:47871 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:51872 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 12:37:00873 }
874
Per Kjellanderb03b6c82021-01-03 09:26:03875 void ResetEncoder(const std::string& payload_name,
876 size_t num_streams,
877 size_t num_temporal_layers,
878 unsigned char num_spatial_layers,
879 bool screenshare,
880 VideoStreamEncoder::BitrateAllocationCallbackType
881 allocation_callback_type =
882 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 14:44:30883 kVideoBitrateAllocationWhenScreenSharing,
884 int num_cores = 1) {
Niels Möller259a4972018-04-05 13:36:51885 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 12:37:00886
887 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:51888 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
889 num_streams, &video_encoder_config);
890 for (auto& layer : video_encoder_config.simulcast_layers) {
891 layer.num_temporal_layers = num_temporal_layers;
892 layer.max_framerate = kDefaultFramerate;
893 }
Erik Språngd7329ca2019-02-21 20:19:53894 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 08:07:11895 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 14:06:52896 video_encoder_config.content_type =
897 screenshare ? VideoEncoderConfig::ContentType::kScreen
898 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 07:28:40899 if (payload_name == "VP9") {
900 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
901 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 13:29:23902 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 07:28:40903 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43904 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
905 vp9_settings);
emircanbbcc3562017-08-18 07:28:40906 }
Erik Språnge4589cb2022-04-06 14:44:30907 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
908 num_cores);
perkj26091b12016-09-01 08:17:40909 }
910
sprang57c2fff2017-01-16 14:24:02911 VideoFrame CreateFrame(int64_t ntp_time_ms,
912 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 17:05:30913 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43914 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 17:05:30915 destruction_event, codec_width_, codec_height_))
916 .set_ntp_time_ms(ntp_time_ms)
917 .set_timestamp_ms(99)
918 .set_rotation(kVideoRotation_0)
919 .build();
perkj26091b12016-09-01 08:17:40920 }
921
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26922 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
923 rtc::Event* destruction_event,
924 int offset_x) const {
Åsa Persson90719572021-04-08 17:05:30925 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43926 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 17:05:30927 destruction_event, codec_width_, codec_height_))
928 .set_ntp_time_ms(ntp_time_ms)
929 .set_timestamp_ms(99)
930 .set_rotation(kVideoRotation_0)
931 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
932 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26933 }
934
sprang57c2fff2017-01-16 14:24:02935 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 12:02:13936 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
937 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 17:05:30938 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 12:02:13939 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 17:05:30940 .set_ntp_time_ms(ntp_time_ms)
941 .set_timestamp_ms(ntp_time_ms)
942 .set_rotation(kVideoRotation_0)
943 .build();
perkj803d97f2016-11-01 18:45:46944 }
945
Evan Shrubsole895556e2020-10-05 07:15:13946 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 17:05:30947 return VideoFrame::Builder()
948 .set_video_frame_buffer(NV12Buffer::Create(width, height))
949 .set_ntp_time_ms(ntp_time_ms)
950 .set_timestamp_ms(ntp_time_ms)
951 .set_rotation(kVideoRotation_0)
952 .build();
Evan Shrubsole895556e2020-10-05 07:15:13953 }
954
Noah Richards51db4212019-06-12 13:59:12955 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
956 rtc::Event* destruction_event,
957 int width,
958 int height) const {
Åsa Persson90719572021-04-08 17:05:30959 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43960 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 17:05:30961 destruction_event, width, height))
962 .set_ntp_time_ms(ntp_time_ms)
963 .set_timestamp_ms(99)
964 .set_rotation(kVideoRotation_0)
965 .build();
Noah Richards51db4212019-06-12 13:59:12966 }
967
Evan Shrubsole895556e2020-10-05 07:15:13968 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
969 rtc::Event* destruction_event,
970 int width,
971 int height) const {
Åsa Persson90719572021-04-08 17:05:30972 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43973 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 17:05:30974 destruction_event, width, height))
975 .set_ntp_time_ms(ntp_time_ms)
976 .set_timestamp_ms(99)
977 .set_rotation(kVideoRotation_0)
978 .build();
Evan Shrubsole895556e2020-10-05 07:15:13979 }
980
Noah Richards51db4212019-06-12 13:59:12981 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
982 rtc::Event* destruction_event) const {
983 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
984 codec_height_);
985 }
986
Åsa Perssonc29cb2c2019-03-25 11:06:59987 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 16:49:07988 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:11989 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 11:06:59990
991 video_source_.IncomingCapturedFrame(
992 CreateFrame(1, codec_width_, codec_height_));
993 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 13:09:05994 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 11:06:59995 }
996
sprang4847ae62017-06-27 14:06:52997 void WaitForEncodedFrame(int64_t expected_ntp_time) {
998 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 12:31:23999 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:521000 }
1001
1002 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
1003 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:231004 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:521005 return ok;
1006 }
1007
1008 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1009 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 12:31:231010 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:521011 }
1012
1013 void ExpectDroppedFrame() {
1014 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 12:31:231015 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:521016 }
1017
1018 bool WaitForFrame(int64_t timeout_ms) {
1019 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:231020 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:521021 return ok;
1022 }
1023
perkj26091b12016-09-01 08:17:401024 class TestEncoder : public test::FakeEncoder {
1025 public:
Tomas Gunnarsson612445e2020-09-21 12:31:231026 explicit TestEncoder(TimeController* time_controller)
1027 : FakeEncoder(time_controller->GetClock()),
1028 time_controller_(time_controller) {
1029 RTC_DCHECK(time_controller_);
1030 }
perkj26091b12016-09-01 08:17:401031
Erik Språngaed30702018-11-05 11:57:171032 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 11:13:321033 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 15:44:421034 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 14:52:331035 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 11:42:181036 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 14:56:001037 info.scaling_settings = VideoEncoder::ScalingSettings(
1038 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 11:42:181039 }
1040 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 11:06:591041 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1042 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 14:27:351043 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 11:06:591044 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 14:27:351045 for (int tid = 0; tid < num_layers; ++tid)
1046 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 11:06:591047 }
1048 }
Erik Språngaed30702018-11-05 11:57:171049 }
Sergey Silkin6456e352019-07-08 15:56:401050
1051 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 08:47:111052 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 15:50:001053 info.apply_alignment_to_all_simulcast_layers =
1054 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 12:56:451055 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-29 22:48:031056 if (is_qp_trusted_.has_value()) {
1057 info.is_qp_trusted = is_qp_trusted_;
1058 }
Erik Språngaed30702018-11-05 11:57:171059 return info;
kthelgason876222f2016-11-29 09:44:111060 }
1061
Erik Språngb7cb7b52019-02-26 14:52:331062 int32_t RegisterEncodeCompleteCallback(
1063 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 11:13:321064 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331065 encoded_image_callback_ = callback;
1066 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1067 }
1068
perkjfa10b552016-10-03 06:45:261069 void ContinueEncode() { continue_encode_event_.Set(); }
1070
1071 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1072 uint32_t timestamp) const {
Markus Handella3765182020-07-08 11:13:321073 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-03 06:45:261074 EXPECT_EQ(timestamp_, timestamp);
1075 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1076 }
1077
kthelgason2fc52542017-03-03 08:24:411078 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 11:13:321079 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 08:24:411080 quality_scaling_ = b;
1081 }
kthelgasonad9010c2017-02-14 08:46:511082
Rasmus Brandt5cad55b2019-12-19 08:47:111083 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 11:13:321084 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 08:47:111085 requested_resolution_alignment_ = requested_resolution_alignment;
1086 }
1087
Åsa Perssonc5a74ff2020-09-20 15:50:001088 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1089 MutexLock lock(&local_mutex_);
1090 apply_alignment_to_all_simulcast_layers_ = b;
1091 }
1092
Mirta Dvornicicccc1b572019-01-15 11:42:181093 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 11:13:321094 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 11:42:181095 is_hardware_accelerated_ = is_hardware_accelerated;
1096 }
1097
Åsa Perssonc29cb2c2019-03-25 11:06:591098 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1099 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 11:13:321100 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:591101 temporal_layers_supported_[spatial_idx] = supported;
1102 }
1103
Sergey Silkin6456e352019-07-08 15:56:401104 void SetResolutionBitrateLimits(
1105 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 11:13:321106 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 15:56:401107 resolution_bitrate_limits_ = thresholds;
1108 }
1109
sprangfe627f32017-03-29 15:24:591110 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 11:13:321111 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 15:24:591112 force_init_encode_failed_ = force_failure;
1113 }
1114
Niels Möller6bb5ab92019-01-11 10:11:101115 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 11:13:321116 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:101117 rate_factor_ = rate_factor;
1118 }
1119
Erik Språngd7329ca2019-02-21 20:19:531120 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 11:13:321121 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:101122 return last_framerate_;
1123 }
1124
Erik Språngd7329ca2019-02-21 20:19:531125 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 11:13:321126 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:261127 return last_update_rect_;
1128 }
1129
Niels Möller87e2d782019-03-07 09:18:231130 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 11:13:321131 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531132 return last_frame_types_;
1133 }
1134
1135 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 09:18:231136 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 14:43:581137 keyframe ? VideoFrameType::kVideoFrameKey
1138 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:531139 {
Markus Handella3765182020-07-08 11:13:321140 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531141 last_frame_types_ = frame_type;
1142 }
Niels Möllerb859b322019-03-07 11:40:011143 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 20:19:531144 }
1145
Sergey Silkin0e42cf72021-03-15 09:12:571146 void InjectEncodedImage(const EncodedImage& image,
1147 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 11:13:321148 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 09:12:571149 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 14:52:331150 }
1151
Mirta Dvornicic97910da2020-07-14 13:29:231152 void SetEncodedImageData(
1153 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 11:13:321154 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 13:29:231155 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 14:30:161156 }
1157
Erik Språngd7329ca2019-02-21 20:19:531158 void ExpectNullFrame() {
Markus Handella3765182020-07-08 11:13:321159 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531160 expect_null_frame_ = true;
1161 }
1162
Erik Språng5056af02019-09-02 13:53:111163 absl::optional<VideoEncoder::RateControlParameters>
1164 GetAndResetLastRateControlSettings() {
1165 auto settings = last_rate_control_settings_;
1166 last_rate_control_settings_.reset();
1167 return settings;
Erik Språngd7329ca2019-02-21 20:19:531168 }
1169
Henrik Boström56db9ff2021-03-24 08:06:451170 int GetLastInputWidth() const {
1171 MutexLock lock(&local_mutex_);
1172 return last_input_width_;
1173 }
1174
1175 int GetLastInputHeight() const {
1176 MutexLock lock(&local_mutex_);
1177 return last_input_height_;
1178 }
1179
Evan Shrubsole895556e2020-10-05 07:15:131180 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1181 MutexLock lock(&local_mutex_);
1182 return last_input_pixel_format_;
1183 }
1184
Evan Shrubsole7c079f62019-09-26 07:55:031185 int GetNumSetRates() const {
Markus Handella3765182020-07-08 11:13:321186 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031187 return num_set_rates_;
1188 }
1189
Evan Shrubsoleb556b082020-10-08 12:56:451190 void SetPreferredPixelFormats(
1191 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1192 pixel_formats) {
1193 MutexLock lock(&local_mutex_);
1194 preferred_pixel_formats_ = std::move(pixel_formats);
1195 }
1196
Qiu Jianlinb54cfde2021-07-29 22:48:031197 void SetIsQpTrusted(absl::optional<bool> trusted) {
1198 MutexLock lock(&local_mutex_);
1199 is_qp_trusted_ = trusted;
1200 }
1201
Erik Språnge4589cb2022-04-06 14:44:301202 VideoCodecComplexity LastEncoderComplexity() {
1203 MutexLock lock(&local_mutex_);
1204 return last_encoder_complexity_;
1205 }
1206
perkjfa10b552016-10-03 06:45:261207 private:
perkj26091b12016-09-01 08:17:401208 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 09:18:231209 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 08:17:401210 {
Markus Handella3765182020-07-08 11:13:321211 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531212 if (expect_null_frame_) {
1213 EXPECT_EQ(input_image.timestamp(), 0u);
1214 EXPECT_EQ(input_image.width(), 1);
1215 last_frame_types_ = *frame_types;
1216 expect_null_frame_ = false;
1217 } else {
1218 EXPECT_GT(input_image.timestamp(), timestamp_);
1219 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1220 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1221 }
perkj26091b12016-09-01 08:17:401222
1223 timestamp_ = input_image.timestamp();
1224 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 18:45:461225 last_input_width_ = input_image.width();
1226 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:261227 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 20:19:531228 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 07:15:131229 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 08:17:401230 }
Niels Möllerb859b322019-03-07 11:40:011231 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 08:17:401232 return result;
1233 }
1234
Niels Möller08ae7ce2020-09-23 13:58:121235 CodecSpecificInfo EncodeHook(
1236 EncodedImage& encoded_image,
1237 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 15:30:361238 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231239 {
1240 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 15:30:361241 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 13:29:231242 }
1243 MutexLock lock(&local_mutex_);
1244 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 15:30:361245 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 13:29:231246 }
Danil Chapovalov2549f172020-08-12 15:30:361247 return codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231248 }
1249
sprangfe627f32017-03-29 15:24:591250 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 12:57:571251 const Settings& settings) override {
1252 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 12:18:341253
Markus Handella3765182020-07-08 11:13:321254 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331255 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 12:18:341256
Erik Språng82fad3d2018-03-21 08:57:231257 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 15:24:591258 // Simulate setting up temporal layers, in order to validate the life
1259 // cycle of these objects.
Elad Aloncde8ab22019-03-20 10:56:201260 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 09:20:091261 frame_buffer_controller_ =
1262 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 15:24:591263 }
Erik Språnge4589cb2022-04-06 14:44:301264
1265 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1266
Erik Språngb7cb7b52019-02-26 14:52:331267 if (force_init_encode_failed_) {
1268 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 15:24:591269 return -1;
Erik Språngb7cb7b52019-02-26 14:52:331270 }
Mirta Dvornicicccc1b572019-01-15 11:42:181271
Erik Språngb7cb7b52019-02-26 14:52:331272 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 15:24:591273 return res;
1274 }
1275
Erik Språngb7cb7b52019-02-26 14:52:331276 int32_t Release() override {
Markus Handella3765182020-07-08 11:13:321277 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331278 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1279 initialized_ = EncoderState::kUninitialized;
1280 return FakeEncoder::Release();
1281 }
1282
Erik Språng16cb8f52019-04-12 11:59:091283 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 11:13:321284 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031285 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 10:11:101286 VideoBitrateAllocation adjusted_rate_allocation;
1287 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1288 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 11:59:091289 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 10:11:101290 adjusted_rate_allocation.SetBitrate(
1291 si, ti,
Erik Språng16cb8f52019-04-12 11:59:091292 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 10:11:101293 rate_factor_));
1294 }
1295 }
1296 }
Erik Språng16cb8f52019-04-12 11:59:091297 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 13:53:111298 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 11:59:091299 RateControlParameters adjusted_paramters = parameters;
1300 adjusted_paramters.bitrate = adjusted_rate_allocation;
1301 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 10:11:101302 }
1303
Tomas Gunnarsson612445e2020-09-21 12:31:231304 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321305 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 14:52:331306 enum class EncoderState {
1307 kUninitialized,
1308 kInitializationFailed,
1309 kInitialized
Markus Handella3765182020-07-08 11:13:321310 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 08:17:401311 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 11:13:321312 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1313 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1314 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1315 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1316 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1317 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 15:50:001318 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1319 false;
Markus Handella3765182020-07-08 11:13:321320 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 13:29:231321 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1322 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 10:56:201323 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 11:13:321324 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:591325 absl::optional<bool>
1326 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 11:13:321327 local_mutex_);
1328 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1329 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1330 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 13:53:111331 absl::optional<VideoEncoder::RateControlParameters>
1332 last_rate_control_settings_;
Markus Handella3765182020-07-08 11:13:321333 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1334 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 09:18:231335 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 20:19:531336 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 11:13:321337 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1338 nullptr;
Evan Shrubsolefb862742020-03-16 15:18:361339 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 15:56:401340 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 11:13:321341 RTC_GUARDED_BY(local_mutex_);
1342 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 07:15:131343 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1344 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 12:56:451345 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1346 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-29 22:48:031347 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 14:44:301348 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1349 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 08:17:401350 };
1351
mflodmancc3d4422017-08-03 15:27:511352 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 08:17:401353 public:
Tomas Gunnarsson612445e2020-09-21 12:31:231354 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1355 : time_controller_(time_controller), test_encoder_(test_encoder) {
1356 RTC_DCHECK(time_controller_);
1357 }
perkj26091b12016-09-01 08:17:401358
perkj26091b12016-09-01 08:17:401359 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 14:06:521360 EXPECT_TRUE(
1361 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1362 }
1363
1364 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1365 int64_t timeout_ms) {
perkj26091b12016-09-01 08:17:401366 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:231367 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 14:06:521368 return false;
perkj26091b12016-09-01 08:17:401369 {
Markus Handella3765182020-07-08 11:13:321370 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121371 timestamp = last_timestamp_;
perkj26091b12016-09-01 08:17:401372 }
1373 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 14:06:521374 return true;
perkj26091b12016-09-01 08:17:401375 }
1376
sprangb1ca0732017-02-01 16:38:121377 void WaitForEncodedFrame(uint32_t expected_width,
1378 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 12:31:231379 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 13:13:561380 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-03 06:53:041381 }
1382
Åsa Perssonc74d8da2017-12-04 13:13:561383 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-03 06:53:041384 uint32_t expected_height) {
sprangb1ca0732017-02-01 16:38:121385 uint32_t width = 0;
1386 uint32_t height = 0;
sprangb1ca0732017-02-01 16:38:121387 {
Markus Handella3765182020-07-08 11:13:321388 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121389 width = last_width_;
1390 height = last_height_;
1391 }
1392 EXPECT_EQ(expected_height, height);
1393 EXPECT_EQ(expected_width, width);
1394 }
1395
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141396 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1397 VideoRotation rotation;
1398 {
Markus Handella3765182020-07-08 11:13:321399 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141400 rotation = last_rotation_;
1401 }
1402 EXPECT_EQ(expected_rotation, rotation);
1403 }
1404
Tomas Gunnarsson612445e2020-09-21 12:31:231405 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 15:02:221406
sprangc5d62e22017-04-03 06:53:041407 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 12:31:231408 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 09:11:551409 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:231410 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 09:11:551411 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:231412 return ret;
sprangc5d62e22017-04-03 06:53:041413 }
1414
perkj26091b12016-09-01 08:17:401415 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 11:13:321416 MutexLock lock(&mutex_);
perkj26091b12016-09-01 08:17:401417 expect_frames_ = false;
1418 }
1419
asaperssonfab67072017-04-04 12:51:491420 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 11:13:321421 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061422 return number_of_reconfigurations_;
1423 }
1424
asaperssonfab67072017-04-04 12:51:491425 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 11:13:321426 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061427 return min_transmit_bitrate_bps_;
1428 }
1429
Erik Språngd7329ca2019-02-21 20:19:531430 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 11:13:321431 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 20:19:531432 num_expected_layers_ = num_layers;
1433 }
1434
Erik Språngb7cb7b52019-02-26 14:52:331435 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 11:13:321436 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331437 return last_capture_time_ms_;
1438 }
1439
Sergey Silkin0e42cf72021-03-15 09:12:571440 const EncodedImage& GetLastEncodedImage() {
1441 MutexLock lock(&mutex_);
1442 return last_encoded_image_;
1443 }
1444
Mirta Dvornicic28f0eb22019-05-28 14:30:161445 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 11:13:321446 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 14:30:161447 return std::move(last_encoded_image_data_);
1448 }
1449
Per Kjellanderdcef6412020-10-07 13:09:051450 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1451 MutexLock lock(&mutex_);
1452 return last_bitrate_allocation_;
1453 }
1454
1455 int number_of_bitrate_allocations() const {
1456 MutexLock lock(&mutex_);
1457 return number_of_bitrate_allocations_;
1458 }
1459
Per Kjellandera9434842020-10-15 15:53:221460 VideoLayersAllocation GetLastVideoLayersAllocation() {
1461 MutexLock lock(&mutex_);
1462 return last_layers_allocation_;
1463 }
1464
1465 int number_of_layers_allocations() const {
1466 MutexLock lock(&mutex_);
1467 return number_of_layers_allocations_;
1468 }
1469
perkj26091b12016-09-01 08:17:401470 private:
sergeyu2cb155a2016-11-04 18:39:291471 Result OnEncodedImage(
1472 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 15:30:361473 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 11:13:321474 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061475 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 09:12:571476 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 14:30:161477 last_encoded_image_data_ = std::vector<uint8_t>(
1478 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 20:19:531479 uint32_t timestamp = encoded_image.Timestamp();
1480 if (last_timestamp_ != timestamp) {
1481 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 12:02:131482 last_width_ = encoded_image._encodedWidth;
1483 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 20:19:531484 } else {
1485 ++num_received_layers_;
Erik Språng7444b192021-06-02 12:02:131486 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1487 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 20:19:531488 }
1489 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 14:52:331490 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141491 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 20:19:531492 if (num_received_layers_ == num_expected_layers_) {
1493 encoded_frame_event_.Set();
1494 }
sprangb1ca0732017-02-01 16:38:121495 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 13:52:061496 }
1497
Rasmus Brandtc402dbe2019-02-04 10:09:461498 void OnEncoderConfigurationChanged(
1499 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 12:10:271500 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 10:09:461501 VideoEncoderConfig::ContentType content_type,
1502 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 11:13:321503 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061504 ++number_of_reconfigurations_;
1505 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1506 }
1507
Per Kjellanderdcef6412020-10-07 13:09:051508 void OnBitrateAllocationUpdated(
1509 const VideoBitrateAllocation& allocation) override {
1510 MutexLock lock(&mutex_);
1511 ++number_of_bitrate_allocations_;
1512 last_bitrate_allocation_ = allocation;
1513 }
1514
Per Kjellandera9434842020-10-15 15:53:221515 void OnVideoLayersAllocationUpdated(
1516 VideoLayersAllocation allocation) override {
1517 MutexLock lock(&mutex_);
1518 ++number_of_layers_allocations_;
1519 last_layers_allocation_ = allocation;
1520 rtc::StringBuilder log;
1521 for (const auto& layer : allocation.active_spatial_layers) {
1522 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1523 << "[";
1524 for (const auto target_bitrate :
1525 layer.target_bitrate_per_temporal_layer) {
1526 log << target_bitrate.kbps() << ",";
1527 }
1528 log << "]";
1529 }
Harald Alvestrand97597c02021-11-04 12:01:231530 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 15:53:221531 }
1532
Tomas Gunnarsson612445e2020-09-21 12:31:231533 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321534 mutable Mutex mutex_;
perkj26091b12016-09-01 08:17:401535 TestEncoder* test_encoder_;
1536 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 09:12:571537 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 14:30:161538 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 16:38:121539 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 14:52:331540 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 16:38:121541 uint32_t last_height_ = 0;
1542 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141543 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 20:19:531544 size_t num_expected_layers_ = 1;
1545 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 08:17:401546 bool expect_frames_ = true;
Per512ecb32016-09-23 13:52:061547 int number_of_reconfigurations_ = 0;
1548 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 13:09:051549 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1550 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 15:53:221551 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1552 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 08:17:401553 };
1554
Sergey Silkin5ee69672019-07-02 12:18:341555 class VideoBitrateAllocatorProxyFactory
1556 : public VideoBitrateAllocatorFactory {
1557 public:
1558 VideoBitrateAllocatorProxyFactory()
1559 : bitrate_allocator_factory_(
1560 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1561
1562 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1563 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 11:13:321564 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341565 codec_config_ = codec;
1566 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1567 }
1568
1569 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 11:13:321570 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341571 return codec_config_;
1572 }
1573
1574 private:
1575 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1576
Markus Handella3765182020-07-08 11:13:321577 mutable Mutex mutex_;
1578 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341579 };
1580
Tomas Gunnarsson612445e2020-09-21 12:31:231581 Clock* clock() { return time_controller_.GetClock(); }
1582 void AdvanceTime(TimeDelta duration) {
1583 time_controller_.AdvanceTime(duration);
1584 }
1585
1586 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1587
1588 protected:
1589 virtual TaskQueueFactory* GetTaskQueueFactory() {
1590 return time_controller_.GetTaskQueueFactory();
1591 }
1592
Jonas Orelandc7f691a2022-03-09 14:12:071593 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 12:31:231594 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 08:17:401595 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 15:41:301596 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 13:52:061597 int codec_width_;
1598 int codec_height_;
sprang4847ae62017-06-27 14:06:521599 int max_framerate_;
perkj26091b12016-09-01 08:17:401600 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 07:07:241601 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 12:18:341602 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-03 06:53:041603 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 08:17:401604 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 12:31:231605 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 15:27:511606 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 08:17:401607};
1608
mflodmancc3d4422017-08-03 15:27:511609TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 16:49:071610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111611 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 07:43:501612 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411613 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 14:06:521614 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 14:53:411615 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 15:27:511616 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401617}
1618
mflodmancc3d4422017-08-03 15:27:511619TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 08:17:401620 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 07:43:501621 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 11:05:491622 // The encoder will cache up to one frame for a short duration. Adding two
1623 // frames means that the first frame will be dropped and the second frame will
1624 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 14:53:411625 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 12:31:231626 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 11:05:491627 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 09:11:551628 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 14:53:411629 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401630
Henrik Boström381d1092020-05-12 16:49:071631 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111632 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401633
Sebastian Janssona3177052018-04-10 11:05:491634 // The pending frame should be received.
sprang4847ae62017-06-27 14:06:521635 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 11:05:491636 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1637
1638 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:511639 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401640}
1641
mflodmancc3d4422017-08-03 15:27:511642TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 16:49:071643 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111644 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 14:53:411645 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521646 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401647
Henrik Boström381d1092020-05-12 16:49:071648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111649 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1650
Sebastian Janssona3177052018-04-10 11:05:491651 // The encoder will cache up to one frame for a short duration. Adding two
1652 // frames means that the first frame will be dropped and the second frame will
1653 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 14:53:411654 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 11:05:491655 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401656
Henrik Boström381d1092020-05-12 16:49:071657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111658 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:521659 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 11:05:491660 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1661 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 15:27:511662 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401663}
1664
mflodmancc3d4422017-08-03 15:27:511665TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 16:49:071666 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111667 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 14:53:411668 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521669 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401670
1671 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 14:53:411672 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 08:17:401673
perkja49cbd32016-09-16 14:53:411674 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521675 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511676 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401677}
1678
mflodmancc3d4422017-08-03 15:27:511679TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 16:49:071680 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111681 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401682
perkja49cbd32016-09-16 14:53:411683 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521684 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401685
mflodmancc3d4422017-08-03 15:27:511686 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401687 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 07:43:501688 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411689 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1690 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401691}
1692
Markus Handell9a478b52021-11-18 15:07:011693TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1694 test::FrameForwarder source;
1695 video_stream_encoder_->SetSource(&source,
1696 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 16:49:071697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111698 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401699
Tomas Gunnarsson612445e2020-09-21 12:31:231700 int dropped_count = 0;
1701 stats_proxy_->SetDroppedFrameCallback(
1702 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1703 ++dropped_count;
1704 });
1705
Markus Handell9a478b52021-11-18 15:07:011706 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1707 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1708 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511709 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 12:31:231710 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 08:17:401711}
1712
Henrik Boström56db9ff2021-03-24 08:06:451713TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 16:49:071714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111715 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121716
1717 rtc::Event frame_destroyed_event;
1718 video_source_.IncomingCapturedFrame(
1719 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 08:06:451720 WaitForEncodedFrame(1);
1721 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1722 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 08:07:111723 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1724 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 13:59:121725 video_stream_encoder_->Stop();
1726}
1727
Henrik Boström56db9ff2021-03-24 08:06:451728TEST_F(VideoStreamEncoderTest,
1729 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 13:59:121730 // Use the cropping factory.
1731 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:431732 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 13:59:121733 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1734 kMaxPayloadLength);
1735 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1736
1737 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 16:49:071738 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111739 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121740 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1741 WaitForEncodedFrame(1);
1742 // The encoder will have been configured once.
1743 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111744 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1745 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 13:59:121746
1747 // Now send in a fake frame that needs to be cropped as the width/height
1748 // aren't divisible by 4 (see CreateEncoderStreams above).
1749 rtc::Event frame_destroyed_event;
1750 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1751 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 08:06:451752 WaitForEncodedFrame(2);
1753 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1754 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 08:07:111755 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1756 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 13:59:121757 video_stream_encoder_->Stop();
1758}
1759
Evan Shrubsole895556e2020-10-05 07:15:131760TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1761 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111762 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 07:15:131763
1764 video_source_.IncomingCapturedFrame(
1765 CreateNV12Frame(1, codec_width_, codec_height_));
1766 WaitForEncodedFrame(1);
1767 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1768 fake_encoder_.GetLastInputPixelFormat());
1769 video_stream_encoder_->Stop();
1770}
1771
Henrik Boström56db9ff2021-03-24 08:06:451772TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 12:56:451773 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111774 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451775
1776 fake_encoder_.SetPreferredPixelFormats({});
1777
1778 rtc::Event frame_destroyed_event;
1779 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1780 1, &frame_destroyed_event, codec_width_, codec_height_));
1781 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451782 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451783 fake_encoder_.GetLastInputPixelFormat());
1784 video_stream_encoder_->Stop();
1785}
1786
Henrik Boström56db9ff2021-03-24 08:06:451787TEST_F(VideoStreamEncoderTest,
1788 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 12:56:451789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111790 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451791
1792 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1793
1794 rtc::Event frame_destroyed_event;
1795 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1796 1, &frame_destroyed_event, codec_width_, codec_height_));
1797 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451798 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451799 fake_encoder_.GetLastInputPixelFormat());
1800 video_stream_encoder_->Stop();
1801}
1802
Henrik Boström56db9ff2021-03-24 08:06:451803TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 12:56:451804 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111805 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451806
1807 // Fake NV12 native frame does not allow mapping to I444.
1808 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1809
1810 rtc::Event frame_destroyed_event;
1811 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1812 1, &frame_destroyed_event, codec_width_, codec_height_));
1813 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451814 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451815 fake_encoder_.GetLastInputPixelFormat());
1816 video_stream_encoder_->Stop();
1817}
1818
Henrik Boström56db9ff2021-03-24 08:06:451819TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 07:15:131820 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111821 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 07:15:131822
1823 rtc::Event frame_destroyed_event;
1824 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1825 1, &frame_destroyed_event, codec_width_, codec_height_));
1826 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451827 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 07:15:131828 fake_encoder_.GetLastInputPixelFormat());
1829 video_stream_encoder_->Stop();
1830}
1831
Ying Wang9b881ab2020-02-07 13:29:321832TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 16:49:071833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111834 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 13:29:321835 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1836 WaitForEncodedFrame(1);
1837
Henrik Boström381d1092020-05-12 16:49:071838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111839 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 13:29:321840 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1841 // frames. Adding two frames means that the first frame will be dropped and
1842 // the second frame will be sent to the encoder.
1843 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1844 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1845 WaitForEncodedFrame(3);
1846 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1847 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1848 WaitForEncodedFrame(5);
1849 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1850 video_stream_encoder_->Stop();
1851}
1852
mflodmancc3d4422017-08-03 15:27:511853TEST_F(VideoStreamEncoderTest,
1854 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 16:49:071855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111856 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 20:37:571857 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061858
1859 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551860 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521861 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571862 // The encoder will have been configured once when the first frame is
1863 // received.
1864 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061865
1866 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:511867 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 13:52:061868 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 15:27:511869 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:471870 kMaxPayloadLength);
Per512ecb32016-09-23 13:52:061871
1872 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551873 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521874 WaitForEncodedFrame(2);
Per21d45d22016-10-30 20:37:571875 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-30 06:25:401876 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-30 05:39:101877
mflodmancc3d4422017-08-03 15:27:511878 video_stream_encoder_->Stop();
perkj26105b42016-09-30 05:39:101879}
1880
mflodmancc3d4422017-08-03 15:27:511881TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 16:49:071882 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111883 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-03 06:45:261884
1885 // Capture a frame and wait for it to synchronize with the encoder thread.
1886 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521887 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571888 // The encoder will have been configured once.
1889 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111890 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1891 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-03 06:45:261892
1893 codec_width_ *= 2;
1894 codec_height_ *= 2;
1895 // Capture a frame with a higher resolution and wait for it to synchronize
1896 // with the encoder thread.
1897 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521898 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 08:07:111899 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1900 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 20:37:571901 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261902
mflodmancc3d4422017-08-03 15:27:511903 video_stream_encoder_->Stop();
perkjfa10b552016-10-03 06:45:261904}
1905
Sergey Silkin443b7ee2019-06-28 10:53:071906TEST_F(VideoStreamEncoderTest,
1907 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 16:49:071908 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111909 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 10:53:071910
1911 // Capture a frame and wait for it to synchronize with the encoder thread.
1912 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1913 WaitForEncodedFrame(1);
1914
1915 VideoEncoderConfig video_encoder_config;
1916 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1917 // Changing the max payload data length recreates encoder.
1918 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1919 kMaxPayloadLength / 2);
1920
1921 // Capture a frame and wait for it to synchronize with the encoder thread.
1922 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1923 WaitForEncodedFrame(2);
1924 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1925
1926 video_stream_encoder_->Stop();
1927}
1928
Sergey Silkin5ee69672019-07-02 12:18:341929TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 16:49:071930 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111931 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 12:18:341932
1933 VideoEncoderConfig video_encoder_config;
1934 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:111935 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1936 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 12:18:341937 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1938 kMaxPayloadLength);
1939
1940 // Capture a frame and wait for it to synchronize with the encoder thread.
1941 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1942 WaitForEncodedFrame(1);
1943 // The encoder will have been configured once when the first frame is
1944 // received.
1945 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111946 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 12:18:341947 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111948 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 12:18:341949 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1950
Sergey Silkin6456e352019-07-08 15:56:401951 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1952 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 08:07:111953 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1954 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 12:18:341955 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1956 kMaxPayloadLength);
1957
1958 // Capture a frame and wait for it to synchronize with the encoder thread.
1959 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1960 WaitForEncodedFrame(2);
1961 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1962 // Bitrate limits have changed - rate allocator should be reconfigured,
1963 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 08:07:111964 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 12:18:341965 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111966 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 12:18:341967 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111968 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 12:18:341969
1970 video_stream_encoder_->Stop();
1971}
1972
Sergey Silkin6456e352019-07-08 15:56:401973TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 13:48:401974 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 16:49:071975 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111976 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401977
Sergey Silkincd02eba2020-01-20 13:48:401978 const uint32_t kMinEncBitrateKbps = 100;
1979 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051980 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 13:48:401981 /*frame_size_pixels=*/codec_width_ * codec_height_,
1982 /*min_start_bitrate_bps=*/0,
1983 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1984 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051985 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1986
Sergey Silkincd02eba2020-01-20 13:48:401987 VideoEncoderConfig video_encoder_config;
1988 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1989 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1990 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1991 (kMinEncBitrateKbps + 1) * 1000;
1992 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1993 kMaxPayloadLength);
1994
1995 // When both encoder and app provide bitrate limits, the intersection of
1996 // provided sets should be used.
1997 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1998 WaitForEncodedFrame(1);
1999 EXPECT_EQ(kMaxEncBitrateKbps,
2000 bitrate_allocator_factory_.codec_config().maxBitrate);
2001 EXPECT_EQ(kMinEncBitrateKbps + 1,
2002 bitrate_allocator_factory_.codec_config().minBitrate);
2003
2004 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2005 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2006 (kMinEncBitrateKbps - 1) * 1000;
2007 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2008 kMaxPayloadLength);
2009 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 14:04:052010 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 13:48:402011 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 14:04:052012 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:402013 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:052014 bitrate_allocator_factory_.codec_config().minBitrate);
2015
Sergey Silkincd02eba2020-01-20 13:48:402016 video_stream_encoder_->Stop();
2017}
2018
2019TEST_F(VideoStreamEncoderTest,
2020 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 16:49:072021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112022 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 13:48:402023
2024 const uint32_t kMinAppBitrateKbps = 100;
2025 const uint32_t kMaxAppBitrateKbps = 200;
2026 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2027 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2028 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2029 /*frame_size_pixels=*/codec_width_ * codec_height_,
2030 /*min_start_bitrate_bps=*/0,
2031 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2032 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2033 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2034
2035 VideoEncoderConfig video_encoder_config;
2036 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2037 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2038 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2039 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 14:04:052040 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2041 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 14:04:052042
Sergey Silkincd02eba2020-01-20 13:48:402043 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2044 WaitForEncodedFrame(1);
2045 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:052046 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:402047 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:052048 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 15:56:402049
2050 video_stream_encoder_->Stop();
2051}
2052
2053TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 14:04:052054 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 16:49:072055 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112056 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:402057
2058 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 14:04:052059 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 15:56:402060 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 14:04:052061 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 15:56:402062 fake_encoder_.SetResolutionBitrateLimits(
2063 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2064
2065 VideoEncoderConfig video_encoder_config;
2066 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2067 video_encoder_config.max_bitrate_bps = 0;
2068 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2069 kMaxPayloadLength);
2070
Sergey Silkin6b2cec12019-08-09 14:04:052071 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 15:56:402072 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2073 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 14:04:052074 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2075 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402076 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2077 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2078
Sergey Silkin6b2cec12019-08-09 14:04:052079 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 15:56:402080 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2081 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 14:04:052082 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2083 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402084 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2085 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2086
Sergey Silkin6b2cec12019-08-09 14:04:052087 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 15:56:402088 // encoder for 360p should be used.
2089 video_source_.IncomingCapturedFrame(
2090 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2091 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 14:04:052092 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2093 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402094 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2095 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2096
Sergey Silkin6b2cec12019-08-09 14:04:052097 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 15:56:402098 // ignored.
2099 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2100 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 14:04:052101 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2102 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402103 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2104 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:052105 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2106 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402107 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2108 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2109
2110 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2111 // for 270p should be used.
2112 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2113 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 14:04:052114 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2115 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402116 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2117 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2118
2119 video_stream_encoder_->Stop();
2120}
2121
Sergey Silkin6b2cec12019-08-09 14:04:052122TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 16:49:072123 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112124 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 14:04:052125
2126 VideoEncoderConfig video_encoder_config;
2127 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2128 video_encoder_config.max_bitrate_bps = 0;
2129 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2130 kMaxPayloadLength);
2131
2132 // Encode 720p frame to get the default encoder target bitrate.
2133 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2134 WaitForEncodedFrame(1);
2135 const uint32_t kDefaultTargetBitrateFor720pKbps =
2136 bitrate_allocator_factory_.codec_config()
2137 .simulcastStream[0]
2138 .targetBitrate;
2139
2140 // Set the max recommended encoder bitrate to something lower than the default
2141 // target bitrate.
2142 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2143 1280 * 720, 10 * 1000, 10 * 1000,
2144 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2145 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2146
2147 // Change resolution to trigger encoder reinitialization.
2148 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2149 WaitForEncodedFrame(2);
2150 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2151 WaitForEncodedFrame(3);
2152
2153 // Ensure the target bitrate is capped by the max bitrate.
2154 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2155 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2156 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2157 .simulcastStream[0]
2158 .targetBitrate *
2159 1000,
2160 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2161
2162 video_stream_encoder_->Stop();
2163}
2164
Åsa Perssona7e34d32021-01-20 14:36:132165TEST_F(VideoStreamEncoderTest,
2166 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2167 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2168 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2169 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2170 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2171 fake_encoder_.SetResolutionBitrateLimits(
2172 {kEncoderLimits270p, kEncoderLimits360p});
2173
2174 // Two streams, highest stream active.
2175 VideoEncoderConfig config;
2176 const int kNumStreams = 2;
2177 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2178 config.max_bitrate_bps = 0;
2179 config.simulcast_layers[0].active = false;
2180 config.simulcast_layers[1].active = true;
2181 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432182 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132183 "VP8", /*max qp*/ 56, /*screencast*/ false,
2184 /*screenshare enabled*/ false);
2185 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2186
2187 // The encoder bitrate limits for 270p should be used.
2188 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2189 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112190 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132191 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112192 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132193 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112194 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132195
2196 // The encoder bitrate limits for 360p should be used.
2197 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2198 EXPECT_FALSE(WaitForFrame(1000));
2199 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112200 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132201 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112202 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132203
2204 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2205 video_source_.IncomingCapturedFrame(
2206 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2207 EXPECT_FALSE(WaitForFrame(1000));
2208 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112209 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132210 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112211 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132212
2213 // Resolution higher than 360p. Encoder limits should be ignored.
2214 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2215 EXPECT_FALSE(WaitForFrame(1000));
2216 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112217 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132218 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112219 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132220 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112221 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132222 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112223 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132224
2225 // Resolution lower than 270p. The encoder limits for 270p should be used.
2226 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2227 EXPECT_FALSE(WaitForFrame(1000));
2228 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112229 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132230 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112231 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132232
2233 video_stream_encoder_->Stop();
2234}
2235
2236TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:512237 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2238 // Two streams, highest stream active.
2239 VideoEncoderConfig config;
2240 const int kNumStreams = 2;
2241 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2242 config.max_bitrate_bps = 0;
2243 config.simulcast_layers[0].active = false;
2244 config.simulcast_layers[1].active = true;
2245 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432246 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 09:39:512247 "VP8", /*max qp*/ 56, /*screencast*/ false,
2248 /*screenshare enabled*/ false);
2249 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2250
2251 // Default bitrate limits for 270p should be used.
2252 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2253 kDefaultLimits270p =
2254 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192255 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:512256 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2257 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112258 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 09:39:512259 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112260 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512261 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112262 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512263
2264 // Default bitrate limits for 360p should be used.
2265 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2266 kDefaultLimits360p =
2267 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192268 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:512269 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2270 EXPECT_FALSE(WaitForFrame(1000));
2271 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112272 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512273 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112274 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512275
2276 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2277 video_source_.IncomingCapturedFrame(
2278 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2279 EXPECT_FALSE(WaitForFrame(1000));
2280 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112281 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512282 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112283 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512284
2285 // Default bitrate limits for 540p should be used.
2286 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2287 kDefaultLimits540p =
2288 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192289 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 09:39:512290 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2291 EXPECT_FALSE(WaitForFrame(1000));
2292 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112293 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512294 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112295 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512296
2297 video_stream_encoder_->Stop();
2298}
2299
2300TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:132301 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2302 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2303 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2304 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2305 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2306 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2307 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2308 fake_encoder_.SetResolutionBitrateLimits(
2309 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2310
2311 // Three streams, middle stream active.
2312 VideoEncoderConfig config;
2313 const int kNumStreams = 3;
2314 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2315 config.simulcast_layers[0].active = false;
2316 config.simulcast_layers[1].active = true;
2317 config.simulcast_layers[2].active = false;
2318 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432319 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132320 "VP8", /*max qp*/ 56, /*screencast*/ false,
2321 /*screenshare enabled*/ false);
2322 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2323
2324 // The encoder bitrate limits for 360p should be used.
2325 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2326 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112327 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132328 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112329 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132330 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112331 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132332
2333 // The encoder bitrate limits for 270p should be used.
2334 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2335 EXPECT_FALSE(WaitForFrame(1000));
2336 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112337 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132338 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112339 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132340
2341 video_stream_encoder_->Stop();
2342}
2343
2344TEST_F(VideoStreamEncoderTest,
2345 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2346 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2347 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2348 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2349 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2350 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2351 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2352 fake_encoder_.SetResolutionBitrateLimits(
2353 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2354
2355 // Three streams, lowest stream active.
2356 VideoEncoderConfig config;
2357 const int kNumStreams = 3;
2358 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2359 config.simulcast_layers[0].active = true;
2360 config.simulcast_layers[1].active = false;
2361 config.simulcast_layers[2].active = false;
2362 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432363 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132364 "VP8", /*max qp*/ 56, /*screencast*/ false,
2365 /*screenshare enabled*/ false);
2366 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2367
2368 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2369 // on lowest stream, limits for 270p should not be used
2370 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2371 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112372 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132373 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112374 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132375 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112376 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132377
2378 video_stream_encoder_->Stop();
2379}
2380
2381TEST_F(VideoStreamEncoderTest,
2382 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2383 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2384 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2385 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2386 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2387 fake_encoder_.SetResolutionBitrateLimits(
2388 {kEncoderLimits270p, kEncoderLimits360p});
2389 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2390
2391 // Two streams, highest stream active.
2392 VideoEncoderConfig config;
2393 const int kNumStreams = 2;
2394 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2395 config.simulcast_layers[0].active = false;
2396 config.simulcast_layers[1].active = true;
2397 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2398 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432399 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132400 "VP8", /*max qp*/ 56, /*screencast*/ false,
2401 /*screenshare enabled*/ false);
2402 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2403
2404 // The encoder bitrate limits for 270p should be used.
2405 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2406 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112407 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132408 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112409 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132410 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112411 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132412
2413 // The max configured bitrate is less than the encoder limit for 360p.
2414 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2415 EXPECT_FALSE(WaitForFrame(1000));
2416 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112417 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132418 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 08:07:112419 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132420
2421 video_stream_encoder_->Stop();
2422}
2423
mflodmancc3d4422017-08-03 15:27:512424TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 18:45:462425 EXPECT_TRUE(video_source_.has_sinks());
2426 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512427 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412428 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:462429 EXPECT_FALSE(video_source_.has_sinks());
2430 EXPECT_TRUE(new_video_source.has_sinks());
2431
mflodmancc3d4422017-08-03 15:27:512432 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462433}
2434
mflodmancc3d4422017-08-03 15:27:512435TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 18:45:462436 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512437 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 18:45:462438 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512439 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462440}
2441
Åsa Perssonc5a74ff2020-09-20 15:50:002442class ResolutionAlignmentTest
2443 : public VideoStreamEncoderTest,
2444 public ::testing::WithParamInterface<
2445 ::testing::tuple<int, std::vector<double>>> {
2446 public:
2447 ResolutionAlignmentTest()
2448 : requested_alignment_(::testing::get<0>(GetParam())),
2449 scale_factors_(::testing::get<1>(GetParam())) {}
2450
2451 protected:
2452 const int requested_alignment_;
2453 const std::vector<double> scale_factors_;
2454};
2455
2456INSTANTIATE_TEST_SUITE_P(
2457 AlignmentAndScaleFactors,
2458 ResolutionAlignmentTest,
2459 ::testing::Combine(
2460 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2461 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2462 std::vector<double>{-1.0, -1.0},
2463 std::vector<double>{-1.0, -1.0, -1.0},
2464 std::vector<double>{4.0, 2.0, 1.0},
2465 std::vector<double>{9999.0, -1.0, 1.0},
2466 std::vector<double>{3.99, 2.01, 1.0},
2467 std::vector<double>{4.9, 1.7, 1.25},
2468 std::vector<double>{10.0, 4.0, 3.0},
2469 std::vector<double>{1.75, 3.5},
2470 std::vector<double>{1.5, 2.5},
2471 std::vector<double>{1.3, 1.0})));
2472
2473TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2474 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 08:47:112475 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 15:50:002476 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2477 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2478
2479 // Fill config with the scaling factor by which to reduce encoding size.
2480 const int num_streams = scale_factors_.size();
2481 VideoEncoderConfig config;
2482 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2483 for (int i = 0; i < num_streams; ++i) {
2484 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2485 }
2486 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432487 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 15:50:002488 "VP8", /*max qp*/ 56, /*screencast*/ false,
2489 /*screenshare enabled*/ false);
2490 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2491
Henrik Boström381d1092020-05-12 16:49:072492 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112493 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2494 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 15:50:002495 // Wait for all layers before triggering event.
2496 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 08:47:112497
2498 // On the 1st frame, we should have initialized the encoder and
2499 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 15:50:002500 int64_t timestamp_ms = kFrameIntervalMs;
2501 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2502 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:112503 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 08:47:112504
2505 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2506 // (It's up the to the encoder to potentially drop the previous frame,
2507 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 15:50:002508 timestamp_ms += kFrameIntervalMs;
2509 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2510 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:112511 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 15:50:002512
Asa Persson606d3cb2021-10-04 08:07:112513 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 15:50:002514 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2515 // Frame size should be a multiple of the requested alignment.
2516 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2517 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2518 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2519 // Aspect ratio should match.
2520 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2521 codec.height * codec.simulcastStream[i].width);
2522 }
Rasmus Brandt5cad55b2019-12-19 08:47:112523
2524 video_stream_encoder_->Stop();
2525}
2526
Jonathan Yubc771b72017-12-09 01:04:292527TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2528 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-14 06:25:222529 const int kWidth = 1280;
2530 const int kHeight = 720;
Jonathan Yubc771b72017-12-09 01:04:292531
2532 // We rely on the automatic resolution adaptation, but we handle framerate
2533 // adaptation manually by mocking the stats proxy.
2534 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-14 06:25:222535
Taylor Brandstetter49fcc102018-05-16 21:20:412536 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 16:49:072537 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112538 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 21:20:412539 video_stream_encoder_->SetSource(&video_source_,
2540 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:392541 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:222542 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292543 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:222544 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2545
Jonathan Yubc771b72017-12-09 01:04:292546 // Adapt down as far as possible.
2547 rtc::VideoSinkWants last_wants;
2548 int64_t t = 1;
2549 int loop_count = 0;
2550 do {
2551 ++loop_count;
2552 last_wants = video_source_.sink_wants();
2553
2554 // Simulate the framerate we've been asked to adapt to.
2555 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2556 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2557 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2558 mock_stats.input_frame_rate = fps;
2559 stats_proxy_->SetMockStats(mock_stats);
2560
2561 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2562 sink_.WaitForEncodedFrame(t);
2563 t += frame_interval_ms;
2564
mflodmancc3d4422017-08-03 15:27:512565 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392566 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292567 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392568 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2569 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292570 } while (video_source_.sink_wants().max_pixel_count <
2571 last_wants.max_pixel_count ||
2572 video_source_.sink_wants().max_framerate_fps <
2573 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222574
Jonathan Yubc771b72017-12-09 01:04:292575 // Verify that we've adapted all the way down.
2576 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222577 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292578 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2579 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-14 06:25:222580 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:292581 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2582 *video_source_.last_sent_height());
2583 EXPECT_EQ(kMinBalancedFramerateFps,
2584 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222585
Jonathan Yubc771b72017-12-09 01:04:292586 // Adapt back up the same number of times we adapted down.
2587 for (int i = 0; i < loop_count - 1; ++i) {
2588 last_wants = video_source_.sink_wants();
2589
2590 // Simulate the framerate we've been asked to adapt to.
2591 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2592 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2593 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2594 mock_stats.input_frame_rate = fps;
2595 stats_proxy_->SetMockStats(mock_stats);
2596
2597 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2598 sink_.WaitForEncodedFrame(t);
2599 t += frame_interval_ms;
2600
Henrik Boström91aa7322020-04-28 10:24:332601 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392602 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292603 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392604 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2605 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292606 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2607 last_wants.max_pixel_count ||
2608 video_source_.sink_wants().max_framerate_fps >
2609 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222610 }
2611
Evan Shrubsole5cd7eb82020-05-25 12:08:392612 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-09 01:04:292613 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222614 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292615 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2616 EXPECT_EQ((loop_count - 1) * 2,
2617 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:222618
mflodmancc3d4422017-08-03 15:27:512619 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222620}
Rasmus Brandt5cad55b2019-12-19 08:47:112621
Evan Shrubsole2e2f6742020-05-14 08:41:152622TEST_F(VideoStreamEncoderTest,
2623 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 08:07:112624 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2625 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392626 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152627
2628 const int kFrameWidth = 1280;
2629 const int kFrameHeight = 720;
2630
2631 int64_t ntp_time = kFrameIntervalMs;
2632
2633 // Force an input frame rate to be available, or the adaptation call won't
2634 // know what framerate to adapt form.
2635 const int kInputFps = 30;
2636 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2637 stats.input_frame_rate = kInputFps;
2638 stats_proxy_->SetMockStats(stats);
2639
2640 video_source_.set_adaptation_enabled(true);
2641 video_stream_encoder_->SetSource(
2642 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 12:08:392643 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152644 video_source_.IncomingCapturedFrame(
2645 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2646 sink_.WaitForEncodedFrame(ntp_time);
2647 ntp_time += kFrameIntervalMs;
2648
2649 // Trigger CPU overuse.
2650 video_stream_encoder_->TriggerCpuOveruse();
2651 video_source_.IncomingCapturedFrame(
2652 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2653 sink_.WaitForEncodedFrame(ntp_time);
2654 ntp_time += kFrameIntervalMs;
2655
2656 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2657 EXPECT_EQ(std::numeric_limits<int>::max(),
2658 video_source_.sink_wants().max_pixel_count);
2659 // Some framerate constraint should be set.
2660 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2661 EXPECT_LT(restricted_fps, kInputFps);
2662 video_source_.IncomingCapturedFrame(
2663 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2664 sink_.WaitForEncodedFrame(ntp_time);
2665 ntp_time += 100;
2666
Henrik Boström2671dac2020-05-19 14:29:092667 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152668 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2669 // Give the encoder queue time to process the change in degradation preference
2670 // by waiting for an encoded frame.
2671 video_source_.IncomingCapturedFrame(
2672 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2673 sink_.WaitForEncodedFrame(ntp_time);
2674 ntp_time += kFrameIntervalMs;
2675
2676 video_stream_encoder_->TriggerQualityLow();
2677 video_source_.IncomingCapturedFrame(
2678 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2679 sink_.WaitForEncodedFrame(ntp_time);
2680 ntp_time += kFrameIntervalMs;
2681
2682 // Some resolution constraint should be set.
2683 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2684 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2685 kFrameWidth * kFrameHeight);
2686 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2687
2688 int pixel_count = video_source_.sink_wants().max_pixel_count;
2689 // Triggering a CPU underuse should not change the sink wants since it has
2690 // not been overused for resolution since we changed degradation preference.
2691 video_stream_encoder_->TriggerCpuUnderuse();
2692 video_source_.IncomingCapturedFrame(
2693 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2694 sink_.WaitForEncodedFrame(ntp_time);
2695 ntp_time += kFrameIntervalMs;
2696 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2697 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2698
Evan Shrubsole64469032020-06-11 08:45:292699 // Change the degradation preference back. CPU underuse should not adapt since
2700 // QP is most limited.
Henrik Boström2671dac2020-05-19 14:29:092701 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152702 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2703 video_source_.IncomingCapturedFrame(
2704 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2705 sink_.WaitForEncodedFrame(ntp_time);
2706 ntp_time += 100;
2707 // Resolution adaptations is gone after changing degradation preference.
2708 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2709 EXPECT_EQ(std::numeric_limits<int>::max(),
2710 video_source_.sink_wants().max_pixel_count);
2711 // The fps adaptation from above is now back.
2712 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2713
2714 // Trigger CPU underuse.
2715 video_stream_encoder_->TriggerCpuUnderuse();
2716 video_source_.IncomingCapturedFrame(
2717 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2718 sink_.WaitForEncodedFrame(ntp_time);
2719 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 08:45:292720 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2721
2722 // Trigger QP underuse, fps should return to normal.
2723 video_stream_encoder_->TriggerQualityHigh();
2724 video_source_.IncomingCapturedFrame(
2725 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2726 sink_.WaitForEncodedFrame(ntp_time);
2727 ntp_time += kFrameIntervalMs;
2728 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 08:41:152729
2730 video_stream_encoder_->Stop();
2731}
2732
mflodmancc3d4422017-08-03 15:27:512733TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 16:49:072734 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112735 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392736 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 18:45:462737
sprangc5d62e22017-04-03 06:53:042738 const int kFrameWidth = 1280;
2739 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:042740
Åsa Persson8c1bf952018-09-13 08:42:192741 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 18:45:462742
kthelgason5e13d412016-12-01 11:59:512743 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042744 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522745 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042746 frame_timestamp += kFrameIntervalMs;
2747
perkj803d97f2016-11-01 18:45:462748 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512749 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462750 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042751 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522752 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042753 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 14:23:482754
asapersson0944a802017-04-07 07:57:582755 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-03 06:53:042756 // wanted resolution.
2757 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2758 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2759 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 08:42:192760 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042761
2762 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 18:45:462763 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 16:49:072764 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412765 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122766 // Give the encoder queue time to process the change in degradation preference
2767 // by waiting for an encoded frame.
2768 new_video_source.IncomingCapturedFrame(
2769 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2770 sink_.WaitForEncodedFrame(frame_timestamp);
2771 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042772 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 12:08:392773 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462774
sprangc5d62e22017-04-03 06:53:042775 // Force an input frame rate to be available, or the adaptation call won't
2776 // know what framerate to adapt form.
asapersson02465b82017-04-10 08:12:522777 const int kInputFps = 30;
sprangc5d62e22017-04-03 06:53:042778 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 08:12:522779 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-03 06:53:042780 stats_proxy_->SetMockStats(stats);
2781
mflodmancc3d4422017-08-03 15:27:512782 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462783 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042784 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522785 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042786 frame_timestamp += kFrameIntervalMs;
2787
2788 // Some framerate constraint should be set.
sprang84a37592017-02-10 15:04:272789 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-03 06:53:042790 EXPECT_EQ(std::numeric_limits<int>::max(),
2791 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522792 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-03 06:53:042793
asapersson02465b82017-04-10 08:12:522794 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 16:49:072795 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2796 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 10:42:122797 // Give the encoder queue time to process the change in degradation preference
2798 // by waiting for an encoded frame.
2799 new_video_source.IncomingCapturedFrame(
2800 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2801 sink_.WaitForEncodedFrame(frame_timestamp);
2802 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 12:08:392803 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-03 06:53:042804
mflodmancc3d4422017-08-03 15:27:512805 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042806 new_video_source.IncomingCapturedFrame(
2807 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522808 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042809 frame_timestamp += kFrameIntervalMs;
2810
2811 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 12:08:392812 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462813
2814 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072815 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412816 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 10:42:122817 // Give the encoder queue time to process the change in degradation preference
2818 // by waiting for an encoded frame.
2819 new_video_source.IncomingCapturedFrame(
2820 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2821 sink_.WaitForEncodedFrame(frame_timestamp);
2822 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042823 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2824 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 15:04:272825 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 08:42:192826 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042827
2828 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072829 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412830 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122831 // Give the encoder queue time to process the change in degradation preference
2832 // by waiting for an encoded frame.
2833 new_video_source.IncomingCapturedFrame(
2834 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2835 sink_.WaitForEncodedFrame(frame_timestamp);
2836 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042837 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2838 EXPECT_EQ(std::numeric_limits<int>::max(),
2839 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522840 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 18:45:462841
mflodmancc3d4422017-08-03 15:27:512842 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462843}
2844
mflodmancc3d4422017-08-03 15:27:512845TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072846 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112847 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 18:45:462848
asaperssonfab67072017-04-04 12:51:492849 const int kWidth = 1280;
2850 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492851 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522852 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 12:51:492853 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2854 EXPECT_FALSE(stats.bw_limited_resolution);
2855 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2856
2857 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512858 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492859 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522860 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 12:51:492861
2862 stats = stats_proxy_->GetStats();
2863 EXPECT_TRUE(stats.bw_limited_resolution);
2864 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2865
2866 // Trigger adapt up.
mflodmancc3d4422017-08-03 15:27:512867 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:492868 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522869 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 12:51:492870
2871 stats = stats_proxy_->GetStats();
2872 EXPECT_FALSE(stats.bw_limited_resolution);
2873 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2874 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2875
mflodmancc3d4422017-08-03 15:27:512876 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 12:51:492877}
2878
mflodmancc3d4422017-08-03 15:27:512879TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072880 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112881 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 12:51:492882
2883 const int kWidth = 1280;
2884 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492885 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522886 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 18:45:462887 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2888 EXPECT_FALSE(stats.cpu_limited_resolution);
2889 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2890
2891 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512892 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492893 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522894 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 18:45:462895
2896 stats = stats_proxy_->GetStats();
2897 EXPECT_TRUE(stats.cpu_limited_resolution);
2898 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2899
2900 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332901 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492902 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522903 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 18:45:462904
2905 stats = stats_proxy_->GetStats();
2906 EXPECT_FALSE(stats.cpu_limited_resolution);
2907 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 12:51:492908 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:462909
mflodmancc3d4422017-08-03 15:27:512910 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462911}
2912
mflodmancc3d4422017-08-03 15:27:512913TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072914 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112915 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112916
asaperssonfab67072017-04-04 12:51:492917 const int kWidth = 1280;
2918 const int kHeight = 720;
2919 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522920 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112921 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182922 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112923 EXPECT_FALSE(stats.cpu_limited_resolution);
2924 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2925
asaperssonfab67072017-04-04 12:51:492926 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512927 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492928 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522929 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112930 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182931 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112932 EXPECT_TRUE(stats.cpu_limited_resolution);
2933 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2934
2935 // Set new source with adaptation still enabled.
2936 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512937 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412938 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112939
asaperssonfab67072017-04-04 12:51:492940 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522941 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112942 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182943 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112944 EXPECT_TRUE(stats.cpu_limited_resolution);
2945 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2946
2947 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412948 video_stream_encoder_->SetSource(&new_video_source,
2949 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 09:44:112950
asaperssonfab67072017-04-04 12:51:492951 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522952 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112953 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182954 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112955 EXPECT_FALSE(stats.cpu_limited_resolution);
2956 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2957
2958 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 15:27:512959 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412960 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112961
asaperssonfab67072017-04-04 12:51:492962 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522963 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112964 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182965 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112966 EXPECT_TRUE(stats.cpu_limited_resolution);
2967 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2968
asaperssonfab67072017-04-04 12:51:492969 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332970 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492971 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522972 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 09:44:112973 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182974 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112975 EXPECT_FALSE(stats.cpu_limited_resolution);
2976 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:522977 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112978
mflodmancc3d4422017-08-03 15:27:512979 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112980}
2981
mflodmancc3d4422017-08-03 15:27:512982TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112984 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112985
asaperssonfab67072017-04-04 12:51:492986 const int kWidth = 1280;
2987 const int kHeight = 720;
2988 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522989 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112990 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112991 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022992 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492993 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112994
2995 // Set new source with adaptation still enabled.
2996 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 21:20:412997 video_stream_encoder_->SetSource(&new_video_source,
2998 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112999
asaperssonfab67072017-04-04 12:51:493000 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523001 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:113002 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:113003 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:023004 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:493005 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:113006
asaperssonfab67072017-04-04 12:51:493007 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:513008 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:493009 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523010 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:113011 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:113012 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:023013 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:493014 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:113015
asaperssonfab67072017-04-04 12:51:493016 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 21:20:413017 video_stream_encoder_->SetSource(&new_video_source,
3018 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:113019
asaperssonfab67072017-04-04 12:51:493020 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523021 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:113022 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:113023 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:023024 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:493025 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:113026
asapersson02465b82017-04-10 08:12:523027 // Disable resolution scaling.
mflodmancc3d4422017-08-03 15:27:513028 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413029 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:113030
asaperssonfab67072017-04-04 12:51:493031 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523032 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:113033 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:113034 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:023035 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:493036 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3037 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 09:44:113038
mflodmancc3d4422017-08-03 15:27:513039 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:113040}
3041
mflodmancc3d4422017-08-03 15:27:513042TEST_F(VideoStreamEncoderTest,
3043 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 16:49:073044 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113045 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 12:29:123046
3047 const int kWidth = 1280;
3048 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 08:42:193049 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 12:29:123050 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 08:42:193051 video_source_.IncomingCapturedFrame(
3052 CreateFrame(timestamp_ms, kWidth, kHeight));
3053 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123054 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3055 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3056 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3057
3058 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:513059 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193060 timestamp_ms += kFrameIntervalMs;
3061 video_source_.IncomingCapturedFrame(
3062 CreateFrame(timestamp_ms, kWidth, kHeight));
3063 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123064 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3065 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3066 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3067
3068 // Trigger overuse.
mflodmancc3d4422017-08-03 15:27:513069 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193070 timestamp_ms += kFrameIntervalMs;
3071 video_source_.IncomingCapturedFrame(
3072 CreateFrame(timestamp_ms, kWidth, kHeight));
3073 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123074 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3075 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3076 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3077
Niels Möller4db138e2018-04-19 07:04:133078 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 12:29:123079 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:133080
3081 VideoEncoderConfig video_encoder_config;
3082 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3083 // Make format different, to force recreation of encoder.
3084 video_encoder_config.video_format.parameters["foo"] = "foo";
3085 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473086 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 08:42:193087 timestamp_ms += kFrameIntervalMs;
3088 video_source_.IncomingCapturedFrame(
3089 CreateFrame(timestamp_ms, kWidth, kHeight));
3090 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123091 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3092 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3093 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3094
mflodmancc3d4422017-08-03 15:27:513095 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 12:29:123096}
3097
mflodmancc3d4422017-08-03 15:27:513098TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 17:39:323099 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 16:49:073100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113101 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 17:39:323102
3103 const int kWidth = 1280;
3104 const int kHeight = 720;
3105 int sequence = 1;
3106
3107 // Enable BALANCED preference, no initial limitation.
3108 test::FrameForwarder source;
3109 video_stream_encoder_->SetSource(&source,
3110 webrtc::DegradationPreference::BALANCED);
3111 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3112 WaitForEncodedFrame(sequence++);
3113 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3114 EXPECT_FALSE(stats.cpu_limited_resolution);
3115 EXPECT_FALSE(stats.cpu_limited_framerate);
3116 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3117
3118 // Trigger CPU overuse, should now adapt down.
3119 video_stream_encoder_->TriggerCpuOveruse();
3120 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3121 WaitForEncodedFrame(sequence++);
3122 stats = stats_proxy_->GetStats();
3123 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3124
3125 // Set new degradation preference should clear restrictions since we changed
3126 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 14:54:213127 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 17:39:323128 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3129 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3130 WaitForEncodedFrame(sequence++);
3131 stats = stats_proxy_->GetStats();
3132 EXPECT_FALSE(stats.cpu_limited_resolution);
3133 EXPECT_FALSE(stats.cpu_limited_framerate);
3134 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3135
3136 // Force an input frame rate to be available, or the adaptation call won't
3137 // know what framerate to adapt from.
3138 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3139 mock_stats.input_frame_rate = 30;
3140 stats_proxy_->SetMockStats(mock_stats);
3141 video_stream_encoder_->TriggerCpuOveruse();
3142 stats_proxy_->ResetMockStats();
3143 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3144 WaitForEncodedFrame(sequence++);
3145
3146 // We have now adapted once.
3147 stats = stats_proxy_->GetStats();
3148 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3149
3150 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 14:54:213151 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3152 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 17:39:323153 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3154 WaitForEncodedFrame(sequence++);
3155 stats = stats_proxy_->GetStats();
3156 EXPECT_FALSE(stats.cpu_limited_resolution);
3157 EXPECT_FALSE(stats.cpu_limited_framerate);
3158 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3159
3160 video_stream_encoder_->Stop();
3161}
3162
3163TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513164 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 16:49:073165 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113166 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 18:45:463167
asapersson0944a802017-04-07 07:57:583168 const int kWidth = 1280;
3169 const int kHeight = 720;
sprang84a37592017-02-10 15:04:273170 int sequence = 1;
perkj803d97f2016-11-01 18:45:463171
asaperssonfab67072017-04-04 12:51:493172 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523173 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463174 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 15:04:273175 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023176 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 15:04:273177 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3178
asapersson02465b82017-04-10 08:12:523179 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 15:27:513180 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:493181 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523182 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 15:04:273183 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 18:45:463184 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023185 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463186 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3187
3188 // Set new source with adaptation still enabled.
3189 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513190 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413191 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:463192
3193 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493194 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523195 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463196 stats = stats_proxy_->GetStats();
3197 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183198 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463199 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3200
sprangc5d62e22017-04-03 06:53:043201 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 15:27:513202 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413203 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 18:45:463204 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493205 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523206 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463207 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-03 06:53:043208 // Not adapted at first.
perkj803d97f2016-11-01 18:45:463209 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183210 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463211 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3212
sprangc5d62e22017-04-03 06:53:043213 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-16 06:40:183214 // know what framerate to adapt from.
sprangc5d62e22017-04-03 06:53:043215 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3216 mock_stats.input_frame_rate = 30;
3217 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513218 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043219 stats_proxy_->ResetMockStats();
3220
3221 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493222 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523223 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043224
3225 // Framerate now adapted.
3226 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:183227 EXPECT_FALSE(stats.cpu_limited_resolution);
3228 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043229 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3230
3231 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 21:20:413232 video_stream_encoder_->SetSource(&new_video_source,
3233 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-03 06:53:043234 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493235 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523236 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043237
3238 stats = stats_proxy_->GetStats();
3239 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023240 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043241 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3242
3243 // Try to trigger overuse. Should not succeed.
3244 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513245 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043246 stats_proxy_->ResetMockStats();
3247
3248 stats = stats_proxy_->GetStats();
3249 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023250 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043251 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3252
3253 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 15:27:513254 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413255 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 12:51:493256 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523257 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463258 stats = stats_proxy_->GetStats();
3259 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023260 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043261 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 18:45:463262
3263 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333264 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:493265 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523266 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463267 stats = stats_proxy_->GetStats();
3268 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023269 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043270 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3271
3272 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 15:27:513273 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413274 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:043275 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493276 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523277 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043278 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 07:01:023279 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-03 06:53:043280 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023281 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043282 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3283
3284 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333285 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:043286 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493287 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523288 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043289 stats = stats_proxy_->GetStats();
3290 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023291 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043292 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:523293 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:463294
mflodmancc3d4422017-08-03 15:27:513295 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:463296}
3297
mflodmancc3d4422017-08-03 15:27:513298TEST_F(VideoStreamEncoderTest,
3299 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 12:51:493300 const int kWidth = 1280;
3301 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073302 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113303 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:113304
asaperssonfab67072017-04-04 12:51:493305 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 12:08:393306 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 09:44:113307
asaperssonfab67072017-04-04 12:51:493308 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523309 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:113310
asaperssonfab67072017-04-04 12:51:493311 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513312 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 11:59:513313
asaperssonfab67072017-04-04 12:51:493314 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523315 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 11:59:513316
kthelgason876222f2016-11-29 09:44:113317 // Expect a scale down.
3318 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 12:51:493319 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 09:44:113320
asapersson02465b82017-04-10 08:12:523321 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 09:44:113322 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513323 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413324 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:113325
asaperssonfab67072017-04-04 12:51:493326 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513327 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:493328 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523329 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:113330
asaperssonfab67072017-04-04 12:51:493331 // Expect no scaling.
sprangc5d62e22017-04-03 06:53:043332 EXPECT_EQ(std::numeric_limits<int>::max(),
3333 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113334
asaperssonfab67072017-04-04 12:51:493335 // Trigger scale up.
mflodmancc3d4422017-08-03 15:27:513336 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:493337 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523338 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:113339
asapersson02465b82017-04-10 08:12:523340 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-03 06:53:043341 EXPECT_EQ(std::numeric_limits<int>::max(),
3342 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113343
mflodmancc3d4422017-08-03 15:27:513344 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:113345}
3346
mflodmancc3d4422017-08-03 15:27:513347TEST_F(VideoStreamEncoderTest,
3348 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523349 const int kWidth = 1280;
3350 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073351 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113352 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523353
Taylor Brandstetter49fcc102018-05-16 21:20:413354 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523355 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513356 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413357 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523358
3359 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523360 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393361 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523362 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3363 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3364
3365 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513366 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393367 EXPECT_THAT(source.sink_wants(),
3368 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523369 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3370 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3371 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3372
3373 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 15:27:513374 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 08:12:523375 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3376 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3377 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3378
mflodmancc3d4422017-08-03 15:27:513379 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523380}
3381
mflodmancc3d4422017-08-03 15:27:513382TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223383 const int kWidth = 1280;
3384 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073385 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113386 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223387
Taylor Brandstetter49fcc102018-05-16 21:20:413388 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223389 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413390 video_stream_encoder_->SetSource(&source,
3391 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223392 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3393 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393394 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223395
3396 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513397 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:393398 EXPECT_THAT(source.sink_wants(),
3399 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223400 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3401 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3402 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3403
3404 // Trigger adapt down for same input resolution, expect no change.
3405 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3406 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:513407 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223408 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3409 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3410 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3411
3412 // Trigger adapt down for larger input resolution, expect no change.
3413 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3414 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:513415 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223416 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3417 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3418 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3419
mflodmancc3d4422017-08-03 15:27:513420 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223421}
3422
mflodmancc3d4422017-08-03 15:27:513423TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 20:55:383424 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3425 const int kWidth = 640;
3426 const int kHeight = 360;
3427 const int64_t kFrameIntervalMs = 150;
3428 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113429 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 20:55:383430
3431 // Enable BALANCED preference, no initial limitation.
3432 AdaptingFrameForwarder source(&time_controller_);
3433 source.set_adaptation_enabled(true);
3434 video_stream_encoder_->SetSource(&source,
3435 webrtc::DegradationPreference::BALANCED);
3436
3437 int64_t timestamp_ms = kFrameIntervalMs;
3438 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3439 sink_.WaitForEncodedFrame(kWidth, kHeight);
3440 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3441 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3442 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3443 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3444
3445 // Trigger adapt down, expect reduced fps (640x360@15fps).
3446 video_stream_encoder_->TriggerQualityLow();
3447 timestamp_ms += kFrameIntervalMs;
3448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3449 sink_.WaitForEncodedFrame(timestamp_ms);
3450 EXPECT_THAT(source.sink_wants(),
3451 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3453 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3454 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3455
3456 // Source requests 270p, expect reduced resolution (480x270@15fps).
3457 source.OnOutputFormatRequest(480, 270);
3458 timestamp_ms += kFrameIntervalMs;
3459 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3460 WaitForEncodedFrame(480, 270);
3461 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3462
3463 // Trigger adapt down, expect reduced fps (480x270@10fps).
3464 video_stream_encoder_->TriggerQualityLow();
3465 timestamp_ms += kFrameIntervalMs;
3466 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3467 sink_.WaitForEncodedFrame(timestamp_ms);
3468 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3469 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3470 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3471 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3472
3473 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3474 source.OnOutputFormatRequest(320, 180);
3475 timestamp_ms += kFrameIntervalMs;
3476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3477 WaitForEncodedFrame(320, 180);
3478 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3479
3480 // Trigger adapt down, expect reduced fps (320x180@7fps).
3481 video_stream_encoder_->TriggerQualityLow();
3482 timestamp_ms += kFrameIntervalMs;
3483 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3484 sink_.WaitForEncodedFrame(timestamp_ms);
3485 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3487 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3488 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3489
3490 // Source requests VGA, expect increased resolution (640x360@7fps).
3491 source.OnOutputFormatRequest(640, 360);
3492 timestamp_ms += kFrameIntervalMs;
3493 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3494 WaitForEncodedFrame(timestamp_ms);
3495 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3496
3497 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3498 video_stream_encoder_->TriggerQualityHigh();
3499 timestamp_ms += kFrameIntervalMs;
3500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3501 WaitForEncodedFrame(timestamp_ms);
3502 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3503 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3505 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3506
3507 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3508 video_stream_encoder_->TriggerQualityHigh();
3509 timestamp_ms += kFrameIntervalMs;
3510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3511 WaitForEncodedFrame(timestamp_ms);
3512 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3513 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3514 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3515 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3516
3517 // Trigger adapt up, expect increased fps (640x360@maxfps).
3518 video_stream_encoder_->TriggerQualityHigh();
3519 timestamp_ms += kFrameIntervalMs;
3520 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3521 WaitForEncodedFrame(timestamp_ms);
3522 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3523 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3525 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3526
3527 video_stream_encoder_->Stop();
3528}
3529
3530TEST_F(VideoStreamEncoderTest,
3531 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3532 const int kWidth = 1280;
3533 const int kHeight = 720;
3534 const int64_t kFrameIntervalMs = 150;
3535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113536 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 20:55:383537
3538 // Enable BALANCED preference, no initial limitation.
3539 AdaptingFrameForwarder source(&time_controller_);
3540 source.set_adaptation_enabled(true);
3541 video_stream_encoder_->SetSource(&source,
3542 webrtc::DegradationPreference::BALANCED);
3543
3544 int64_t timestamp_ms = kFrameIntervalMs;
3545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3546 sink_.WaitForEncodedFrame(kWidth, kHeight);
3547 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3548 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3549 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3550 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3551
3552 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3553 video_stream_encoder_->TriggerQualityLow();
3554 timestamp_ms += kFrameIntervalMs;
3555 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3556 sink_.WaitForEncodedFrame(timestamp_ms);
3557 EXPECT_THAT(source.sink_wants(),
3558 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3560 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3561 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3562
3563 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3564 video_stream_encoder_->TriggerQualityLow();
3565 timestamp_ms += kFrameIntervalMs;
3566 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3567 sink_.WaitForEncodedFrame(timestamp_ms);
3568 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3569 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3570 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3571 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3572
3573 // Trigger adapt down, expect reduced fps (640x360@15fps).
3574 video_stream_encoder_->TriggerQualityLow();
3575 timestamp_ms += kFrameIntervalMs;
3576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3577 WaitForEncodedFrame(timestamp_ms);
3578 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3581 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3582
3583 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3584 source.OnOutputFormatRequest(320, 180);
3585 timestamp_ms += kFrameIntervalMs;
3586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3587 WaitForEncodedFrame(320, 180);
3588 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3589 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3590
3591 // Trigger adapt down, expect reduced fps (320x180@7fps).
3592 video_stream_encoder_->TriggerCpuOveruse();
3593 timestamp_ms += kFrameIntervalMs;
3594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3595 WaitForEncodedFrame(timestamp_ms);
3596 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3599 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3600 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3601 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3602 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3603
3604 // Source requests HD, expect increased resolution (640x360@7fps).
3605 source.OnOutputFormatRequest(1280, 720);
3606 timestamp_ms += kFrameIntervalMs;
3607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3608 WaitForEncodedFrame(timestamp_ms);
3609 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3610 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3611
3612 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3613 video_stream_encoder_->TriggerCpuUnderuse();
3614 timestamp_ms += kFrameIntervalMs;
3615 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3616 WaitForEncodedFrame(timestamp_ms);
3617 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3619 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3620 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3621 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3622 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3623 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3624
3625 // Trigger adapt up, expect increased fps (640x360@maxfps).
3626 video_stream_encoder_->TriggerQualityHigh();
3627 video_stream_encoder_->TriggerCpuUnderuse();
3628 timestamp_ms += kFrameIntervalMs;
3629 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3630 WaitForEncodedFrame(timestamp_ms);
3631 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3632 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3633 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3634 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3635 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3636 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3637 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3638
3639 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3640 video_stream_encoder_->TriggerQualityHigh();
3641 video_stream_encoder_->TriggerCpuUnderuse();
3642 timestamp_ms += kFrameIntervalMs;
3643 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3644 WaitForEncodedFrame(timestamp_ms);
3645 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3647 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3648 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3649 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3650 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3651 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3652
3653 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3654 video_stream_encoder_->TriggerQualityHigh();
3655 video_stream_encoder_->TriggerCpuUnderuse();
3656 timestamp_ms += kFrameIntervalMs;
3657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3658 WaitForEncodedFrame(timestamp_ms);
3659 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3661 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3663 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3664 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3665 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3666
3667 video_stream_encoder_->Stop();
3668}
3669
3670TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513671 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523672 const int kWidth = 1280;
3673 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073674 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113675 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523676
Taylor Brandstetter49fcc102018-05-16 21:20:413677 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523678 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513679 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413680 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523681
3682 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523683 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393684 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523685 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3686 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3687
3688 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333689 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393690 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523691 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3692 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3693
mflodmancc3d4422017-08-03 15:27:513694 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523695}
3696
mflodmancc3d4422017-08-03 15:27:513697TEST_F(VideoStreamEncoderTest,
3698 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 08:12:523699 const int kWidth = 1280;
3700 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073701 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113702 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523703
Taylor Brandstetter49fcc102018-05-16 21:20:413704 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523705 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513706 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413707 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 08:12:523708
3709 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523710 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393711 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183712 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523713 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3714
3715 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333716 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393717 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523719 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3720
mflodmancc3d4422017-08-03 15:27:513721 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523722}
3723
mflodmancc3d4422017-08-03 15:27:513724TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223725 const int kWidth = 1280;
3726 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073727 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113728 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223729
Taylor Brandstetter49fcc102018-05-16 21:20:413730 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223731 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413732 video_stream_encoder_->SetSource(&source,
3733 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223734
3735 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3736 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393737 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223738 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3739 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3740 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3741
3742 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513743 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393744 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223745 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3746 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3747 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3748
mflodmancc3d4422017-08-03 15:27:513749 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223750}
3751
mflodmancc3d4422017-08-03 15:27:513752TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-16 06:40:183753 const int kWidth = 1280;
3754 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073755 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113756 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:183757
Taylor Brandstetter49fcc102018-05-16 21:20:413758 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-16 06:40:183759 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413760 video_stream_encoder_->SetSource(&source,
3761 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-16 06:40:183762
3763 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3764 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393765 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183766 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3767 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3768 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3769
3770 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513771 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393772 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183773 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3774 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3775 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3776
mflodmancc3d4422017-08-03 15:27:513777 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183778}
3779
mflodmancc3d4422017-08-03 15:27:513780TEST_F(VideoStreamEncoderTest,
3781 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523782 const int kWidth = 1280;
3783 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113785 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523786
Taylor Brandstetter49fcc102018-05-16 21:20:413787 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233788 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 08:12:523789 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513790 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413791 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523792
3793 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523794 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393795 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3797 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3798
3799 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513800 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 08:12:523801 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523802 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393803 EXPECT_THAT(source.sink_wants(),
3804 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523805 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3806 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3807
3808 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513809 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393810 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523811 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3812 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3813 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3814
mflodmancc3d4422017-08-03 15:27:513815 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523816}
3817
mflodmancc3d4422017-08-03 15:27:513818TEST_F(VideoStreamEncoderTest,
3819 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-16 06:40:183820 const int kWidth = 1280;
3821 const int kHeight = 720;
3822 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 16:49:073823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:183825
3826 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3827 stats.input_frame_rate = kInputFps;
3828 stats_proxy_->SetMockStats(stats);
3829
Taylor Brandstetter49fcc102018-05-16 21:20:413830 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-16 06:40:183831 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3832 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393833 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183834
3835 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513836 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-16 06:40:183837 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3838 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393839 EXPECT_THAT(video_source_.sink_wants(),
3840 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-16 06:40:183841
Taylor Brandstetter49fcc102018-05-16 21:20:413842 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-16 06:40:183843 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 14:29:093844 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:413845 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:123846 // Give the encoder queue time to process the change in degradation preference
3847 // by waiting for an encoded frame.
3848 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3849 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 12:08:393850 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183851
3852 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 15:27:513853 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 10:42:123854 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3855 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 12:08:393856 EXPECT_THAT(new_video_source.sink_wants(),
3857 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-16 06:40:183858
3859 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513860 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393861 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183862
mflodmancc3d4422017-08-03 15:27:513863 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183864}
3865
mflodmancc3d4422017-08-03 15:27:513866TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 08:47:313867 const int kWidth = 1280;
3868 const int kHeight = 720;
3869 const size_t kNumFrames = 10;
3870
Henrik Boström381d1092020-05-12 16:49:073871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113872 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 11:59:513873
asaperssond0de2952017-04-21 08:47:313874 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 15:58:543875 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 08:47:313876 video_source_.set_adaptation_enabled(true);
3877
3878 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3879 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3880
3881 int downscales = 0;
3882 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 08:42:193883 video_source_.IncomingCapturedFrame(
3884 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3885 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 08:47:313886
asaperssonfab67072017-04-04 12:51:493887 // Trigger scale down.
asaperssond0de2952017-04-21 08:47:313888 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 15:27:513889 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-03 06:53:043890 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 08:47:313891
3892 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3893 ++downscales;
3894
3895 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3896 EXPECT_EQ(downscales,
3897 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3898 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 11:59:513899 }
mflodmancc3d4422017-08-03 15:27:513900 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313901}
3902
mflodmancc3d4422017-08-03 15:27:513903TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:313904 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3905 const int kWidth = 1280;
3906 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113908 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:313909
Taylor Brandstetter49fcc102018-05-16 21:20:413910 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233911 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:313912 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513913 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413914 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:313915
Åsa Persson8c1bf952018-09-13 08:42:193916 int64_t timestamp_ms = kFrameIntervalMs;
3917 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523918 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393919 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313920 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3921 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3922
3923 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513924 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193925 timestamp_ms += kFrameIntervalMs;
3926 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3927 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393928 EXPECT_THAT(source.sink_wants(),
3929 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313930 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3931 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3932
3933 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333934 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193935 timestamp_ms += kFrameIntervalMs;
3936 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523937 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393938 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313939 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3940 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3941
3942 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513943 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193944 timestamp_ms += kFrameIntervalMs;
3945 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3946 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393947 EXPECT_THAT(source.sink_wants(),
3948 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313949 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3950 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3951
3952 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333953 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193954 timestamp_ms += kFrameIntervalMs;
3955 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-16 06:40:183956 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393957 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313958 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3959 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3960
mflodmancc3d4422017-08-03 15:27:513961 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313962}
3963
mflodmancc3d4422017-08-03 15:27:513964TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-14 06:25:223965 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3966 const int kWidth = 1280;
3967 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073968 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113969 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223970
Taylor Brandstetter49fcc102018-05-16 21:20:413971 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233972 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:223973 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:413974 video_stream_encoder_->SetSource(&source,
3975 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223976
Åsa Persson8c1bf952018-09-13 08:42:193977 int64_t timestamp_ms = kFrameIntervalMs;
3978 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223979 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393980 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223981 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3982 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3983
3984 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513985 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193986 timestamp_ms += kFrameIntervalMs;
3987 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3988 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393989 EXPECT_THAT(source.sink_wants(),
3990 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223991 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3992 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3993
3994 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513995 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193996 timestamp_ms += kFrameIntervalMs;
3997 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223998 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393999 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:224000 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4001 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4002
4003 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:514004 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194005 timestamp_ms += kFrameIntervalMs;
4006 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4007 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:394008 EXPECT_THAT(source.sink_wants(),
4009 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:224010 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4011 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4012
4013 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:514014 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194015 timestamp_ms += kFrameIntervalMs;
4016 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:224017 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:394018 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:224019 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4020 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4021
mflodmancc3d4422017-08-03 15:27:514022 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224023}
4024
Sergey Silkin41c650b2019-10-14 11:12:194025TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4026 fake_encoder_.SetResolutionBitrateLimits(
4027 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4028
Henrik Boström381d1092020-05-12 16:49:074029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074030 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4031 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4032 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4033 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194034
4035 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234036 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:194037 source.set_adaptation_enabled(true);
4038 video_stream_encoder_->SetSource(
4039 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4040
4041 // Insert 720p frame.
4042 int64_t timestamp_ms = kFrameIntervalMs;
4043 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4044 WaitForEncodedFrame(1280, 720);
4045
4046 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 16:49:074047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074048 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4049 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4050 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4051 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194052 video_stream_encoder_->TriggerQualityLow();
4053
4054 // Insert 720p frame. It should be downscaled and encoded.
4055 timestamp_ms += kFrameIntervalMs;
4056 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4057 WaitForEncodedFrame(960, 540);
4058
4059 // Trigger adapt up. Higher resolution should not be requested duo to lack
4060 // of bitrate.
4061 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:394062 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 11:12:194063
4064 // Increase bitrate.
Henrik Boström381d1092020-05-12 16:49:074065 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074066 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4067 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4068 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4069 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194070
4071 // Trigger adapt up. Higher resolution should be requested.
4072 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:394073 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 11:12:194074
4075 video_stream_encoder_->Stop();
4076}
4077
4078TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4079 fake_encoder_.SetResolutionBitrateLimits(
4080 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4081
4082 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 16:49:074083 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074084 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4085 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4086 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4087 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194088
4089 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234090 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:194091 source.set_adaptation_enabled(true);
4092 video_stream_encoder_->SetSource(
4093 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4094
4095 // Insert 720p frame. It should be dropped and lower resolution should be
4096 // requested.
4097 int64_t timestamp_ms = kFrameIntervalMs;
4098 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4099 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:094100 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 11:12:194101
4102 // Insert 720p frame. It should be downscaled and encoded.
4103 timestamp_ms += kFrameIntervalMs;
4104 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4105 WaitForEncodedFrame(960, 540);
4106
4107 video_stream_encoder_->Stop();
4108}
4109
Åsa Perssonb67c44c2019-09-24 13:25:324110class BalancedDegradationTest : public VideoStreamEncoderTest {
4111 protected:
4112 void SetupTest() {
4113 // Reset encoder for field trials to take effect.
4114 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:114115 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 13:25:324116
4117 // Enable BALANCED preference.
4118 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 13:13:044119 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4120 }
4121
Asa Persson606d3cb2021-10-04 08:07:114122 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 16:49:074123 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114124 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 13:25:324125 }
4126
Åsa Persson45b176f2019-09-30 09:19:054127 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 13:25:324128 timestamp_ms_ += kFrameIntervalMs;
4129 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 09:19:054130 }
4131
4132 void InsertFrameAndWaitForEncoded() {
4133 InsertFrame();
Åsa Perssonb67c44c2019-09-24 13:25:324134 sink_.WaitForEncodedFrame(timestamp_ms_);
4135 }
4136
4137 const int kWidth = 640; // pixels:640x360=230400
4138 const int kHeight = 360;
4139 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4140 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:234141 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 13:25:324142};
4143
Evan Shrubsolea1c77f62020-08-10 09:01:064144TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 14:12:074145 test::ScopedKeyValueConfig field_trials(
4146 field_trials_,
Åsa Perssonb67c44c2019-09-24 13:25:324147 "WebRTC-Video-BalancedDegradationSettings/"
4148 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4149 SetupTest();
4150
4151 // Force input frame rate.
4152 const int kInputFps = 24;
4153 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4154 stats.input_frame_rate = kInputFps;
4155 stats_proxy_->SetMockStats(stats);
4156
Åsa Persson45b176f2019-09-30 09:19:054157 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394158 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324159
Evan Shrubsolea1c77f62020-08-10 09:01:064160 // Trigger adapt down, expect scaled down framerate and resolution,
4161 // since Fps diff (input-requested:0) < threshold.
4162 video_stream_encoder_->TriggerQualityLow();
4163 EXPECT_THAT(source_.sink_wants(),
4164 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 13:25:324165
4166 video_stream_encoder_->Stop();
4167}
4168
Evan Shrubsolea1c77f62020-08-10 09:01:064169TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 14:12:074170 test::ScopedKeyValueConfig field_trials(
4171 field_trials_,
Åsa Perssonb67c44c2019-09-24 13:25:324172 "WebRTC-Video-BalancedDegradationSettings/"
4173 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4174 SetupTest();
4175
4176 // Force input frame rate.
4177 const int kInputFps = 25;
4178 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4179 stats.input_frame_rate = kInputFps;
4180 stats_proxy_->SetMockStats(stats);
4181
Åsa Persson45b176f2019-09-30 09:19:054182 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394183 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324184
Evan Shrubsolea1c77f62020-08-10 09:01:064185 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4186 // Fps diff (input-requested:1) == threshold.
4187 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:394188 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 13:25:324189
4190 video_stream_encoder_->Stop();
4191}
4192
4193TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 14:12:074194 test::ScopedKeyValueConfig field_trials(
4195 field_trials_,
Åsa Perssonb67c44c2019-09-24 13:25:324196 "WebRTC-Video-BalancedDegradationSettings/"
4197 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4198 SetupTest();
4199
4200 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4201
Åsa Persson45b176f2019-09-30 09:19:054202 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394203 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324204
4205 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4206 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:394207 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 13:25:324208
4209 video_stream_encoder_->Stop();
4210}
4211
Åsa Perssonccfb3402019-09-25 13:13:044212TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 14:12:074213 test::ScopedKeyValueConfig field_trials(
4214 field_trials_,
Åsa Persson1b247f12019-08-14 15:26:394215 "WebRTC-Video-BalancedDegradationSettings/"
4216 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 13:13:044217 SetupTest();
Åsa Persson1b247f12019-08-14 15:26:394218
Asa Persson606d3cb2021-10-04 08:07:114219 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4220 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4221 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 15:26:394222
Åsa Persson45b176f2019-09-30 09:19:054223 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394224 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 15:26:394225 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4226
4227 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4228 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054229 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394230 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 15:26:394231 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4232
4233 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4234 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054235 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544236 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:394237 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4238
Åsa Persson30ab0152019-08-27 10:22:334239 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4240 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054241 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544242 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 13:13:044243 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 10:22:334244 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4245
4246 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 15:26:394247 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054248 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334249 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 15:26:394250
Åsa Persson30ab0152019-08-27 10:22:334251 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114252 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 15:26:394253 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054254 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:044255 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 10:22:334256 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4257
4258 video_stream_encoder_->Stop();
4259}
4260
Åsa Perssonccfb3402019-09-25 13:13:044261TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 09:19:054262 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 14:12:074263 test::ScopedKeyValueConfig field_trials(
4264 field_trials_,
Åsa Persson45b176f2019-09-30 09:19:054265 "WebRTC-Video-BalancedDegradationSettings/"
4266 "pixels:57600|129600|230400,fps:7|24|24/");
4267 SetupTest();
Asa Persson606d3cb2021-10-04 08:07:114268 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 09:19:054269
Evan Shrubsole5cd7eb82020-05-25 12:08:394270 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 09:19:054271
4272 // Insert frame, expect scaled down:
4273 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4274 InsertFrame();
4275 EXPECT_FALSE(WaitForFrame(1000));
4276 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4277 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4278
4279 // Insert frame, expect scaled down:
4280 // resolution (320x180@24fps).
4281 InsertFrame();
4282 EXPECT_FALSE(WaitForFrame(1000));
4283 EXPECT_LT(source_.sink_wants().max_pixel_count,
4284 source_.last_wants().max_pixel_count);
4285 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4286
4287 // Frame should not be dropped (min pixels per frame reached).
4288 InsertFrameAndWaitForEncoded();
4289
4290 video_stream_encoder_->Stop();
4291}
4292
4293TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334294 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 14:12:074295 test::ScopedKeyValueConfig field_trials(
4296 field_trials_,
Åsa Persson30ab0152019-08-27 10:22:334297 "WebRTC-Video-BalancedDegradationSettings/"
4298 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044299 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334300
Asa Persson606d3cb2021-10-04 08:07:114301 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4302 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4303 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 10:22:334304
Åsa Persson45b176f2019-09-30 09:19:054305 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394306 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334307 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4308
4309 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4310 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054311 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394312 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334313 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4314
4315 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4316 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054317 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544318 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334319 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4320
4321 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4322 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054323 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544324 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:394325 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4326
Åsa Persson30ab0152019-08-27 10:22:334327 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4328 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054329 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544330 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334331 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4332
4333 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4334 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054335 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334336 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4337
4338 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114339 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334340 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054341 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544342 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334343 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4344
4345 video_stream_encoder_->Stop();
4346}
4347
Åsa Perssonccfb3402019-09-25 13:13:044348TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334349 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 14:12:074350 test::ScopedKeyValueConfig field_trials(
4351 field_trials_,
Åsa Persson30ab0152019-08-27 10:22:334352 "WebRTC-Video-BalancedDegradationSettings/"
4353 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044354 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334355
Asa Persson606d3cb2021-10-04 08:07:114356 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4357 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4358 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4359 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4360 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334361
Åsa Persson45b176f2019-09-30 09:19:054362 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394363 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334364 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4365
4366 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4367 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054368 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394369 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334370 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4371
4372 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4373 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054374 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544375 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334376 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4377
4378 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4379 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054380 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544381 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334382 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4383
4384 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4385 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054386 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334387 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4388
4389 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114390 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334391 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054392 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544393 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334394 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4395
4396 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114397 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 10:22:334398 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054399 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334400 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4401
4402 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114403 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334404 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054405 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544406 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334407 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4408
Åsa Persson1b247f12019-08-14 15:26:394409 video_stream_encoder_->Stop();
4410}
4411
mflodmancc3d4422017-08-03 15:27:514412TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:314413 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4414 const int kWidth = 1280;
4415 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:074416 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114417 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:314418
Taylor Brandstetter49fcc102018-05-16 21:20:414419 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234420 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:314421 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:514422 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414423 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:314424
Åsa Persson8c1bf952018-09-13 08:42:194425 int64_t timestamp_ms = kFrameIntervalMs;
4426 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524427 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:394428 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314429 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4430 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4431 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4432 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4433
4434 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 15:27:514435 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194436 timestamp_ms += kFrameIntervalMs;
4437 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4438 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:394439 EXPECT_THAT(source.sink_wants(),
4440 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:314441 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4442 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4443 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4444 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4445
4446 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 15:27:514447 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194448 timestamp_ms += kFrameIntervalMs;
4449 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4450 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544451 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314452 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4453 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4454 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4455 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4456
Jonathan Yubc771b72017-12-09 01:04:294457 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 15:27:514458 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194459 timestamp_ms += kFrameIntervalMs;
4460 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4461 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544462 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314463 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4464 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294465 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:314466 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4467
Jonathan Yubc771b72017-12-09 01:04:294468 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 15:27:514469 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194470 timestamp_ms += kFrameIntervalMs;
4471 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4472 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544473 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294474 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 08:47:314475 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4476 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4477 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4478 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4479
Jonathan Yubc771b72017-12-09 01:04:294480 // Trigger quality adapt down, expect no change (min resolution reached).
4481 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194482 timestamp_ms += kFrameIntervalMs;
4483 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4484 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544485 EXPECT_THAT(source.sink_wants(), FpsMax());
4486 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-09 01:04:294487 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4488 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4489 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4490 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4491
Evan Shrubsole64469032020-06-11 08:45:294492 // Trigger quality adapt up, expect upscaled resolution (480x270).
4493 video_stream_encoder_->TriggerQualityHigh();
4494 timestamp_ms += kFrameIntervalMs;
4495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4496 WaitForEncodedFrame(timestamp_ms);
4497 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4498 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4499 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4500 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4501 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4502
4503 // Trigger quality and cpu adapt up since both are most limited, expect
4504 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 10:24:334505 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294506 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194507 timestamp_ms += kFrameIntervalMs;
4508 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4509 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544510 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294511 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4512 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4513 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294514 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:294515
Evan Shrubsole64469032020-06-11 08:45:294516 // Trigger quality and cpu adapt up since both are most limited, expect
4517 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 10:24:334518 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294519 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194520 timestamp_ms += kFrameIntervalMs;
4521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4522 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544523 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314524 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 08:45:294525 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 08:47:314526 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:294527 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4528 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314529
Evan Shrubsole64469032020-06-11 08:45:294530 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4531 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 10:24:334532 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:194533 timestamp_ms += kFrameIntervalMs;
4534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4535 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544536 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 08:47:314537 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4538 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294539 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294540 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314541
4542 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 15:27:514543 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194544 timestamp_ms += kFrameIntervalMs;
4545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524546 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:544547 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:394548 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314549 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4550 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294551 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294552 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 11:59:514553
mflodmancc3d4422017-08-03 15:27:514554 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 11:59:514555}
4556
mflodmancc3d4422017-08-03 15:27:514557TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 12:51:494558 const int kWidth = 640;
4559 const int kHeight = 360;
perkj803d97f2016-11-01 18:45:464560
Henrik Boström381d1092020-05-12 16:49:074561 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114562 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:524563
perkj803d97f2016-11-01 18:45:464564 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494565 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524566 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 18:45:464567 }
4568
mflodmancc3d4422017-08-03 15:27:514569 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:464570 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494571 video_source_.IncomingCapturedFrame(CreateFrame(
4572 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524573 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 18:45:464574 }
4575
mflodmancc3d4422017-08-03 15:27:514576 video_stream_encoder_->Stop();
4577 video_stream_encoder_.reset();
perkj803d97f2016-11-01 18:45:464578 stats_proxy_.reset();
sprangf8ee65e2017-02-28 16:49:334579
Ying Wangef3998f2019-12-09 12:06:534580 EXPECT_METRIC_EQ(
4581 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4582 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 18:45:464583 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4584}
4585
mflodmancc3d4422017-08-03 15:27:514586TEST_F(VideoStreamEncoderTest,
4587 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 16:49:074588 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114589 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 09:01:064590 const int kWidth = 640;
4591 const int kHeight = 360;
4592
Taylor Brandstetter49fcc102018-05-16 21:20:414593 video_stream_encoder_->SetSource(&video_source_,
4594 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 09:01:064595
4596 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4597 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524598 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 09:01:064599 }
4600
mflodmancc3d4422017-08-03 15:27:514601 video_stream_encoder_->Stop();
4602 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 09:01:064603 stats_proxy_.reset();
4604
4605 EXPECT_EQ(0,
4606 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4607}
4608
Per Kjellanderdcef6412020-10-07 13:09:054609TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4610 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034611 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:054612 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 14:24:024613
4614 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 10:32:224615 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 08:07:114616 SimulcastRateAllocator(fake_encoder_.config())
4617 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 13:16:284618 kDefaultFps));
sprang57c2fff2017-01-16 14:24:024619
Henrik Boström381d1092020-05-12 16:49:074620 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114621 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 14:24:024622
sprang57c2fff2017-01-16 14:24:024623 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234624 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4625 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054626 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4627 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4628
Erik Språngd7329ca2019-02-21 20:19:534629 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 15:44:424630 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 12:31:234631 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024632
Per Kjellanderdcef6412020-10-07 13:09:054633 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 14:24:024634 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234635 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4636 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054637 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 12:31:234638 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024639
Per Kjellanderdcef6412020-10-07 13:09:054640 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 12:31:234641 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 09:28:414642 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 20:19:534643 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234644 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4645 WaitForEncodedFrame(CurrentTimeMs());
4646 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 20:19:534647 }
Per Kjellanderdcef6412020-10-07 13:09:054648 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 20:19:534649
mflodmancc3d4422017-08-03 15:27:514650 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 14:24:024651}
4652
Per Kjellanderf86cf4c2020-12-30 14:27:354653TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 15:53:224654 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034655 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:224656 kVideoLayersAllocation);
4657
4658 const int kDefaultFps = 30;
4659
4660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114661 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:224662
4663 video_source_.IncomingCapturedFrame(
4664 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4665 WaitForEncodedFrame(CurrentTimeMs());
4666 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4667 VideoLayersAllocation last_layer_allocation =
4668 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 08:07:114669 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 15:53:224670 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4671
4672 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 15:44:424673 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 15:53:224674 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 08:07:114675 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 15:53:224676 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4677
Erik Språng9d69cbe2020-10-22 15:44:424678 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 15:53:224679 int number_of_layers_allocation = 1;
4680 const int64_t start_time_ms = CurrentTimeMs();
4681 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4682 video_source_.IncomingCapturedFrame(
4683 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4684 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 15:53:224685 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4686 number_of_layers_allocation = sink_.number_of_layers_allocations();
4687 VideoLayersAllocation new_allocation =
4688 sink_.GetLastVideoLayersAllocation();
4689 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4690 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4691 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4692 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4693 .target_bitrate_per_temporal_layer,
4694 last_layer_allocation.active_spatial_layers[0]
4695 .target_bitrate_per_temporal_layer);
4696 last_layer_allocation = new_allocation;
4697 }
4698 }
4699 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4700 video_stream_encoder_->Stop();
4701}
4702
4703TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134704 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354705 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4706 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4707 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354708 VideoEncoderConfig video_encoder_config;
4709 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4710 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114711 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354712 video_encoder_config.content_type =
4713 VideoEncoderConfig::ContentType::kRealtimeVideo;
4714 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434715 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354716 VideoEncoder::GetDefaultVp8Settings());
4717 for (auto& layer : video_encoder_config.simulcast_layers) {
4718 layer.num_temporal_layers = 2;
4719 }
4720 // Simulcast layers are used for enabling/disabling streams.
4721 video_encoder_config.simulcast_layers[0].active = true;
4722 video_encoder_config.simulcast_layers[1].active = false;
4723 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034724 ConfigureEncoder(std::move(video_encoder_config),
4725 VideoStreamEncoder::BitrateAllocationCallbackType::
4726 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354727
4728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114729 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354730
4731 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4732 WaitForEncodedFrame(CurrentTimeMs());
4733 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4734 VideoLayersAllocation last_layer_allocation =
4735 sink_.GetLastVideoLayersAllocation();
4736
4737 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4738 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4739 .target_bitrate_per_temporal_layer,
4740 SizeIs(2));
4741 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4742 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4743 video_stream_encoder_->Stop();
4744}
4745
4746TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134747 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354748 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4749 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4750 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354751 VideoEncoderConfig video_encoder_config;
4752 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4753 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114754 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354755 video_encoder_config.content_type =
4756 VideoEncoderConfig::ContentType::kRealtimeVideo;
4757 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434758 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354759 VideoEncoder::GetDefaultVp8Settings());
4760 for (auto& layer : video_encoder_config.simulcast_layers) {
4761 layer.num_temporal_layers = 2;
4762 }
4763 // Simulcast layers are used for enabling/disabling streams.
4764 video_encoder_config.simulcast_layers[0].active = true;
4765 video_encoder_config.simulcast_layers[1].active = false;
4766 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034767 ConfigureEncoder(std::move(video_encoder_config),
4768 VideoStreamEncoder::BitrateAllocationCallbackType::
4769 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354770
4771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114772 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354773
4774 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4775 WaitForEncodedFrame(CurrentTimeMs());
4776 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4777 VideoLayersAllocation last_layer_allocation =
4778 sink_.GetLastVideoLayersAllocation();
4779
4780 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4781 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4782 .target_bitrate_per_temporal_layer,
4783 SizeIs(2));
4784 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4785
4786 video_stream_encoder_->Stop();
4787}
4788
4789TEST_F(VideoStreamEncoderTest,
4790 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4791 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4792 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354793 VideoEncoderConfig video_encoder_config;
4794 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4795 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114796 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354797 video_encoder_config.content_type =
4798 VideoEncoderConfig::ContentType::kRealtimeVideo;
4799 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4800 vp9_settings.numberOfSpatialLayers = 2;
4801 vp9_settings.numberOfTemporalLayers = 2;
4802 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4803 vp9_settings.automaticResizeOn = false;
4804 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434805 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354806 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034807 ConfigureEncoder(std::move(video_encoder_config),
4808 VideoStreamEncoder::BitrateAllocationCallbackType::
4809 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354810
4811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114812 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354813
4814 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4815 WaitForEncodedFrame(CurrentTimeMs());
4816 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4817 VideoLayersAllocation last_layer_allocation =
4818 sink_.GetLastVideoLayersAllocation();
4819
4820 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4821 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4822 .target_bitrate_per_temporal_layer,
4823 SizeIs(2));
4824 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4825 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4826 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4827 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4828 .target_bitrate_per_temporal_layer,
4829 SizeIs(2));
4830 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4831 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4832 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4833
4834 // Since full SVC is used, expect the top layer to utilize the full target
4835 // rate.
4836 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4837 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114838 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354839 video_stream_encoder_->Stop();
4840}
4841
4842TEST_F(VideoStreamEncoderTest,
4843 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4844 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4845 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 14:27:354846 VideoEncoderConfig video_encoder_config;
4847 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4848 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114849 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354850 video_encoder_config.content_type =
4851 VideoEncoderConfig::ContentType::kRealtimeVideo;
4852 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4853 vp9_settings.numberOfSpatialLayers = 2;
4854 vp9_settings.numberOfTemporalLayers = 2;
4855 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4856 vp9_settings.automaticResizeOn = false;
4857 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434858 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354859 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034860 ConfigureEncoder(std::move(video_encoder_config),
4861 VideoStreamEncoder::BitrateAllocationCallbackType::
4862 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354863
4864 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114865 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354866
4867 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4868 WaitForEncodedFrame(CurrentTimeMs());
4869 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4870 VideoLayersAllocation last_layer_allocation =
4871 sink_.GetLastVideoLayersAllocation();
4872
4873 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4874 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4875 .target_bitrate_per_temporal_layer,
4876 SizeIs(1));
4877 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4878 .target_bitrate_per_temporal_layer,
4879 SizeIs(1));
4880 // Since full SVC is used, expect the top layer to utilize the full target
4881 // rate.
4882 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4883 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:114884 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354885 video_stream_encoder_->Stop();
4886}
4887
4888TEST_F(VideoStreamEncoderTest,
4889 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4890 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4891 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354892 VideoEncoderConfig video_encoder_config;
4893 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4894 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114895 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354896 video_encoder_config.content_type =
4897 VideoEncoderConfig::ContentType::kRealtimeVideo;
4898 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4899 vp9_settings.numberOfSpatialLayers = 2;
4900 vp9_settings.numberOfTemporalLayers = 2;
4901 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4902 vp9_settings.automaticResizeOn = false;
4903 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434904 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354905 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034906 ConfigureEncoder(std::move(video_encoder_config),
4907 VideoStreamEncoder::BitrateAllocationCallbackType::
4908 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354909
4910 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114911 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354912
4913 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4914 WaitForEncodedFrame(CurrentTimeMs());
4915 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4916 VideoLayersAllocation last_layer_allocation =
4917 sink_.GetLastVideoLayersAllocation();
4918
4919 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4920 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4921 .target_bitrate_per_temporal_layer,
4922 SizeIs(2));
4923 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4924 .target_bitrate_per_temporal_layer,
4925 SizeIs(2));
4926 // Since KSVC is, spatial layers are independend except on key frames.
4927 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4928 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114929 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354930 video_stream_encoder_->Stop();
4931}
4932
4933TEST_F(VideoStreamEncoderTest,
4934 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4935 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4936 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4937 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354938 VideoEncoderConfig video_encoder_config;
4939 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4940 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114941 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354942 video_encoder_config.content_type =
4943 VideoEncoderConfig::ContentType::kRealtimeVideo;
4944 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4945 vp9_settings.numberOfSpatialLayers = 3;
4946 vp9_settings.numberOfTemporalLayers = 2;
4947 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4948 vp9_settings.automaticResizeOn = false;
4949 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434950 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354951 vp9_settings);
4952 // Simulcast layers are used for enabling/disabling streams.
4953 video_encoder_config.simulcast_layers.resize(3);
4954 video_encoder_config.simulcast_layers[0].active = false;
4955 video_encoder_config.simulcast_layers[1].active = true;
4956 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034957 ConfigureEncoder(std::move(video_encoder_config),
4958 VideoStreamEncoder::BitrateAllocationCallbackType::
4959 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354960
4961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114962 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354963
4964 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4965 WaitForEncodedFrame(CurrentTimeMs());
4966 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4967 VideoLayersAllocation last_layer_allocation =
4968 sink_.GetLastVideoLayersAllocation();
4969
4970 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4971 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4972 .target_bitrate_per_temporal_layer,
4973 SizeIs(2));
4974 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4975 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4976
4977 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4978 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4979 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4980 .target_bitrate_per_temporal_layer,
4981 SizeIs(2));
4982 // Since full SVC is used, expect the top layer to utilize the full target
4983 // rate.
4984 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4985 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114986 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354987 video_stream_encoder_->Stop();
4988}
4989
4990TEST_F(VideoStreamEncoderTest,
4991 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4992 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4993 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4994 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354995 VideoEncoderConfig video_encoder_config;
4996 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4997 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114998 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354999 video_encoder_config.content_type =
5000 VideoEncoderConfig::ContentType::kRealtimeVideo;
5001 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5002 vp9_settings.numberOfSpatialLayers = 3;
5003 vp9_settings.numberOfTemporalLayers = 2;
5004 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5005 vp9_settings.automaticResizeOn = false;
5006 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435007 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:355008 vp9_settings);
5009 // Simulcast layers are used for enabling/disabling streams.
5010 video_encoder_config.simulcast_layers.resize(3);
5011 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:035012 ConfigureEncoder(std::move(video_encoder_config),
5013 VideoStreamEncoder::BitrateAllocationCallbackType::
5014 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:355015
5016 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115017 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:355018
5019 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5020 WaitForEncodedFrame(CurrentTimeMs());
5021 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5022 VideoLayersAllocation last_layer_allocation =
5023 sink_.GetLastVideoLayersAllocation();
5024
5025 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5026 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5027 .target_bitrate_per_temporal_layer,
5028 SizeIs(2));
5029 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5030 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5031
5032 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5033 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5034 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5035 .target_bitrate_per_temporal_layer,
5036 SizeIs(2));
5037 video_stream_encoder_->Stop();
5038}
5039
5040TEST_F(VideoStreamEncoderTest,
5041 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5042 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5043 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5044 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:355045 VideoEncoderConfig video_encoder_config;
5046 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5047 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:115048 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:355049 video_encoder_config.content_type =
5050 VideoEncoderConfig::ContentType::kRealtimeVideo;
5051 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5052 vp9_settings.numberOfSpatialLayers = 3;
5053 vp9_settings.numberOfTemporalLayers = 2;
5054 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5055 vp9_settings.automaticResizeOn = false;
5056 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435057 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:355058 vp9_settings);
5059 // Simulcast layers are used for enabling/disabling streams.
5060 video_encoder_config.simulcast_layers.resize(3);
5061 video_encoder_config.simulcast_layers[0].active = false;
5062 video_encoder_config.simulcast_layers[1].active = false;
5063 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:035064 ConfigureEncoder(std::move(video_encoder_config),
5065 VideoStreamEncoder::BitrateAllocationCallbackType::
5066 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:355067
5068 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115069 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:355070
5071 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5072 WaitForEncodedFrame(CurrentTimeMs());
5073 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5074 VideoLayersAllocation last_layer_allocation =
5075 sink_.GetLastVideoLayersAllocation();
5076
5077 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5078 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5079 .target_bitrate_per_temporal_layer,
5080 SizeIs(2));
5081 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5082 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5083 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5084 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:115085 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:355086 video_stream_encoder_->Stop();
5087}
5088
5089TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5090 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:035091 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 14:27:355092 kVideoLayersAllocation);
5093 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115094 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:355095
5096 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5097 WaitForEncodedFrame(CurrentTimeMs());
5098 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5099 VideoLayersAllocation last_layer_allocation =
5100 sink_.GetLastVideoLayersAllocation();
5101
5102 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5103 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5104 .target_bitrate_per_temporal_layer,
5105 SizeIs(1));
5106 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5107 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:115108 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:355109 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5110 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5111 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5112 video_stream_encoder_->Stop();
5113}
5114
5115TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 15:53:225116 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5117 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035118 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:225119 kVideoLayersAllocation);
5120
5121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115122 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:225123
5124 video_source_.IncomingCapturedFrame(
5125 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5126 WaitForEncodedFrame(CurrentTimeMs());
5127 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5128 VideoLayersAllocation last_layer_allocation =
5129 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 08:07:115130 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 15:53:225131 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5132 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5133 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:115134 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 15:53:225135
5136 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115137 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5138 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:225139 video_source_.IncomingCapturedFrame(
5140 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5141 WaitForEncodedFrame(CurrentTimeMs());
5142
5143 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5144 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5145 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5146 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5147 .target_bitrate_per_temporal_layer[0],
5148 DataRate::Zero());
5149
5150 video_stream_encoder_->Stop();
5151}
5152
Per Kjellander4190ce92020-12-15 16:24:555153TEST_F(VideoStreamEncoderTest,
5154 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5155 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035156 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 16:24:555157 kVideoLayersAllocation);
5158
5159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115160 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5161 0, 0, 0);
Per Kjellander4190ce92020-12-15 16:24:555162
5163 video_source_.IncomingCapturedFrame(
5164 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5165 WaitForEncodedFrame(CurrentTimeMs());
5166 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5167 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5168 SizeIs(2));
5169 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5170 codec_width_);
5171 EXPECT_EQ(
5172 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5173 codec_height_);
5174
5175 video_source_.IncomingCapturedFrame(
5176 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5177 WaitForEncodedFrame(CurrentTimeMs());
5178 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5179 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5180 SizeIs(2));
5181 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5182 codec_width_ / 2);
5183 EXPECT_EQ(
5184 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5185 codec_height_ / 2);
5186
5187 video_stream_encoder_->Stop();
5188}
5189
Åsa Perssonc29cb2c2019-03-25 11:06:595190TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5191 // 2 TLs configured, temporal layers supported by encoder.
5192 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 13:09:055193 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035194 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055195 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595196 fake_encoder_.SetTemporalLayersSupported(0, true);
5197
5198 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 08:07:115199 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 11:06:595200 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:465201 kNumTemporalLayers, /*temporal_id*/ 0,
5202 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 08:07:115203 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 11:06:595204 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:465205 kNumTemporalLayers, /*temporal_id*/ 1,
5206 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595207 VideoBitrateAllocation expected_bitrate;
5208 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5209 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5210
5211 VerifyAllocatedBitrate(expected_bitrate);
5212 video_stream_encoder_->Stop();
5213}
5214
5215TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5216 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 13:09:055217 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035218 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055219 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595220 fake_encoder_.SetTemporalLayersSupported(0, false);
5221
5222 // Temporal layers not supported by the encoder.
5223 // Total bitrate should be at ti:0.
5224 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 08:07:115225 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 11:06:595226
5227 VerifyAllocatedBitrate(expected_bitrate);
5228 video_stream_encoder_->Stop();
5229}
5230
5231TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 14:12:075232 webrtc::test::ScopedKeyValueConfig field_trials(
5233 field_trials_,
Per Kjellanderdcef6412020-10-07 13:09:055234 "WebRTC-Video-QualityScalerSettings/"
5235 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5236 // Reset encoder for field trials to take effect.
5237 ConfigureEncoder(video_encoder_config_.Copy());
5238
Åsa Perssonc29cb2c2019-03-25 11:06:595239 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 13:09:055240 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035241 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055242 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595243 fake_encoder_.SetTemporalLayersSupported(0, true);
5244 fake_encoder_.SetTemporalLayersSupported(1, false);
5245
5246 const int kS0Bps = 150000;
5247 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:465248 kS0Bps *
5249 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5250 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595251 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:465252 kS0Bps *
5253 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5254 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 08:07:115255 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 11:06:595256 // Temporal layers not supported by si:1.
5257 VideoBitrateAllocation expected_bitrate;
5258 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5259 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5260 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5261
5262 VerifyAllocatedBitrate(expected_bitrate);
5263 video_stream_encoder_->Stop();
5264}
5265
Niels Möller7dc26b72017-12-06 09:27:485266TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5267 const int kFrameWidth = 1280;
5268 const int kFrameHeight = 720;
5269 const int kFramerate = 24;
5270
Henrik Boström381d1092020-05-12 16:49:075271 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115272 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485273 test::FrameForwarder source;
5274 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415275 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485276
5277 // Insert a single frame, triggering initial configuration.
5278 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5279 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5280
5281 EXPECT_EQ(
5282 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5283 kDefaultFramerate);
5284
5285 // Trigger reconfigure encoder (without resetting the entire instance).
5286 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515287 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5288 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:115289 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 09:27:485290 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475291 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485292 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5293
5294 // Detector should be updated with fps limit from codec config.
5295 EXPECT_EQ(
5296 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5297 kFramerate);
5298
5299 // Trigger overuse, max framerate should be reduced.
5300 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5301 stats.input_frame_rate = kFramerate;
5302 stats_proxy_->SetMockStats(stats);
5303 video_stream_encoder_->TriggerCpuOveruse();
5304 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5305 int adapted_framerate =
5306 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5307 EXPECT_LT(adapted_framerate, kFramerate);
5308
5309 // Trigger underuse, max framerate should go back to codec configured fps.
5310 // Set extra low fps, to make sure it's actually reset, not just incremented.
5311 stats = stats_proxy_->GetStats();
5312 stats.input_frame_rate = adapted_framerate / 2;
5313 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335314 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485315 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5316 EXPECT_EQ(
5317 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5318 kFramerate);
5319
5320 video_stream_encoder_->Stop();
5321}
5322
5323TEST_F(VideoStreamEncoderTest,
5324 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5325 const int kFrameWidth = 1280;
5326 const int kFrameHeight = 720;
5327 const int kLowFramerate = 15;
5328 const int kHighFramerate = 25;
5329
Henrik Boström381d1092020-05-12 16:49:075330 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115331 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485332 test::FrameForwarder source;
5333 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415334 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485335
5336 // Trigger initial configuration.
5337 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515338 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5339 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 08:07:115340 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 09:27:485341 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 06:57:515342 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 07:51:475343 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485344 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5345
5346 EXPECT_EQ(
5347 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5348 kLowFramerate);
5349
5350 // Trigger overuse, max framerate should be reduced.
5351 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5352 stats.input_frame_rate = kLowFramerate;
5353 stats_proxy_->SetMockStats(stats);
5354 video_stream_encoder_->TriggerCpuOveruse();
5355 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5356 int adapted_framerate =
5357 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5358 EXPECT_LT(adapted_framerate, kLowFramerate);
5359
5360 // Reconfigure the encoder with a new (higher max framerate), max fps should
5361 // still respect the adaptation.
Åsa Persson17107062020-10-08 06:57:515362 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 09:27:485363 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5364 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475365 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485366 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5367
5368 EXPECT_EQ(
5369 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5370 adapted_framerate);
5371
5372 // Trigger underuse, max framerate should go back to codec configured fps.
5373 stats = stats_proxy_->GetStats();
5374 stats.input_frame_rate = adapted_framerate;
5375 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335376 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485377 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5378 EXPECT_EQ(
5379 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5380 kHighFramerate);
5381
5382 video_stream_encoder_->Stop();
5383}
5384
mflodmancc3d4422017-08-03 15:27:515385TEST_F(VideoStreamEncoderTest,
5386 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 11:21:075387 const int kFrameWidth = 1280;
5388 const int kFrameHeight = 720;
5389 const int kFramerate = 24;
5390
Henrik Boström381d1092020-05-12 16:49:075391 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115392 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 11:21:075393 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:515394 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415395 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 11:21:075396
5397 // Trigger initial configuration.
5398 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515399 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5400 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:115401 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 11:21:075402 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 15:27:515403 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475404 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:515405 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 11:21:075406
Niels Möller7dc26b72017-12-06 09:27:485407 EXPECT_EQ(
5408 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5409 kFramerate);
sprangfda496a2017-06-15 11:21:075410
5411 // Trigger overuse, max framerate should be reduced.
5412 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5413 stats.input_frame_rate = kFramerate;
5414 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 15:27:515415 video_stream_encoder_->TriggerCpuOveruse();
5416 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:485417 int adapted_framerate =
5418 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 11:21:075419 EXPECT_LT(adapted_framerate, kFramerate);
5420
5421 // Change degradation preference to not enable framerate scaling. Target
5422 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 16:49:075423 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:415424 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 09:27:485425 EXPECT_EQ(
5426 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5427 kFramerate);
sprangfda496a2017-06-15 11:21:075428
mflodmancc3d4422017-08-03 15:27:515429 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 11:21:075430}
5431
mflodmancc3d4422017-08-03 15:27:515432TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495433 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075435 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5436 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5437 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495438 const int kWidth = 640;
5439 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225440
asaperssonfab67072017-04-04 12:51:495441 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225442
5443 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525444 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225445
5446 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095447 EXPECT_TRUE_WAIT(
5448 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 15:02:225449
sprangc5d62e22017-04-03 06:53:045450 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 15:02:225451
asaperssonfab67072017-04-04 12:51:495452 // Next frame is scaled.
kthelgason2bc68642017-02-07 15:02:225453 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:495454 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 15:02:225455
5456 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525457 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225458
Henrik Boström2671dac2020-05-19 14:29:095459 EXPECT_TRUE_WAIT(
5460 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 15:02:225461
mflodmancc3d4422017-08-03 15:27:515462 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225463}
5464
mflodmancc3d4422017-08-03 15:27:515465TEST_F(VideoStreamEncoderTest,
5466 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495467 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075468 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075469 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5470 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5471 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495472 const int kWidth = 640;
5473 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225474
5475 // We expect the n initial frames to get dropped.
5476 int i;
5477 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 12:51:495478 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525479 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225480 }
5481 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 12:51:495482 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525483 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 15:02:225484
5485 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 12:51:495486 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:225487
mflodmancc3d4422017-08-03 15:27:515488 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225489}
5490
mflodmancc3d4422017-08-03 15:27:515491TEST_F(VideoStreamEncoderTest,
5492 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 12:51:495493 const int kWidth = 640;
5494 const int kHeight = 360;
Henrik Boström381d1092020-05-12 16:49:075495 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115496 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 15:02:225497
5498 // Set degradation preference.
mflodmancc3d4422017-08-03 15:27:515499 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415500 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 15:02:225501
asaperssonfab67072017-04-04 12:51:495502 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225503 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525504 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 15:02:225505
mflodmancc3d4422017-08-03 15:27:515506 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225507}
5508
mflodmancc3d4422017-08-03 15:27:515509TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 12:51:495510 const int kWidth = 640;
5511 const int kHeight = 360;
kthelgasonad9010c2017-02-14 08:46:515512 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:135513
5514 VideoEncoderConfig video_encoder_config;
5515 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5516 // Make format different, to force recreation of encoder.
5517 video_encoder_config.video_format.parameters["foo"] = "foo";
5518 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475519 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 16:49:075520 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115521 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:185522
kthelgasonb83797b2017-02-14 19:57:255523 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 21:20:415524 video_stream_encoder_->SetSource(&video_source_,
5525 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 08:46:515526
asaperssonfab67072017-04-04 12:51:495527 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 08:46:515528 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525529 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 08:46:515530
mflodmancc3d4422017-08-03 15:27:515531 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 08:46:515532 fake_encoder_.SetQualityScaling(true);
5533}
5534
Åsa Persson139f4dc2019-08-02 07:29:585535TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 14:12:075536 webrtc::test::ScopedKeyValueConfig field_trials(
5537 field_trials_,
Åsa Persson139f4dc2019-08-02 07:29:585538 "WebRTC-Video-QualityScalerSettings/"
5539 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5540 // Reset encoder for field trials to take effect.
5541 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:115542 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5543 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 07:29:585544 const int kWidth = 640;
5545 const int kHeight = 360;
5546
Henrik Boström381d1092020-05-12 16:49:075547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115548 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585549 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5550 // Frame should not be dropped.
5551 WaitForEncodedFrame(1);
5552
Henrik Boström381d1092020-05-12 16:49:075553 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075554 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5555 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5556 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585557 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5558 // Frame should not be dropped.
5559 WaitForEncodedFrame(2);
5560
Henrik Boström381d1092020-05-12 16:49:075561 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075562 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5563 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5564 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585565 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5566 // Expect to drop this frame, the wait should time out.
5567 ExpectDroppedFrame();
5568
5569 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095570 EXPECT_TRUE_WAIT(
5571 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 07:29:585572 video_stream_encoder_->Stop();
5573}
5574
Evan Shrubsolee3da1d32020-08-14 13:58:335575TEST_F(VideoStreamEncoderTest,
5576 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 14:12:075577 webrtc::test::ScopedKeyValueConfig field_trials(
5578 field_trials_,
Evan Shrubsolee3da1d32020-08-14 13:58:335579 "WebRTC-Video-QualityScalerSettings/"
5580 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5581 fake_encoder_.SetQualityScaling(false);
5582 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:115583 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5584 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 13:58:335585 const int kWidth = 640;
5586 const int kHeight = 360;
5587
5588 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115589 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 13:58:335590 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5591 // Frame should not be dropped.
5592 WaitForEncodedFrame(1);
5593
5594 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5595 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5596 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5597 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5598 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5599 // Frame should not be dropped.
5600 WaitForEncodedFrame(2);
5601
5602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5603 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5604 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5605 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5606 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5607 // Not dropped since quality scaling is disabled.
5608 WaitForEncodedFrame(3);
5609
5610 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 11:12:125611 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 13:58:335612 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5613
5614 video_stream_encoder_->Stop();
5615}
5616
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475617TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 08:07:115618 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475619 // Set simulcast.
5620 ResetEncoder("VP8", 3, 1, 1, false);
5621 fake_encoder_.SetQualityScaling(true);
5622 const int kWidth = 1280;
5623 const int kHeight = 720;
5624 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115625 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475626 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5627 // Frame should not be dropped.
5628 WaitForEncodedFrame(1);
5629
5630 // Trigger QVGA "singlecast"
5631 // Update the config.
5632 VideoEncoderConfig video_encoder_config;
5633 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5634 &video_encoder_config);
Åsa Persson7f354f82021-02-04 14:52:155635 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:435636 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 14:52:155637 "VP8", /*max qp*/ 56, /*screencast*/ false,
5638 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475639 for (auto& layer : video_encoder_config.simulcast_layers) {
5640 layer.num_temporal_layers = 1;
5641 layer.max_framerate = kDefaultFramerate;
5642 }
Asa Persson606d3cb2021-10-04 08:07:115643 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475644 video_encoder_config.content_type =
5645 VideoEncoderConfig::ContentType::kRealtimeVideo;
5646
5647 video_encoder_config.simulcast_layers[0].active = true;
5648 video_encoder_config.simulcast_layers[1].active = false;
5649 video_encoder_config.simulcast_layers[2].active = false;
5650
5651 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5652 kMaxPayloadLength);
5653 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5654
5655 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5656 // Frame should not be dropped.
5657 WaitForEncodedFrame(2);
5658
5659 // Trigger HD "singlecast"
5660 video_encoder_config.simulcast_layers[0].active = false;
5661 video_encoder_config.simulcast_layers[1].active = false;
5662 video_encoder_config.simulcast_layers[2].active = true;
5663
5664 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5665 kMaxPayloadLength);
5666 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5667
5668 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5669 // Frame should be dropped because of initial frame drop.
5670 ExpectDroppedFrame();
5671
5672 // Expect the sink_wants to specify a scaled frame.
5673 EXPECT_TRUE_WAIT(
5674 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5675 video_stream_encoder_->Stop();
5676}
5677
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085678TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 08:07:115679 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085680 // Set simulcast.
5681 ResetEncoder("VP9", 1, 1, 3, false);
5682 fake_encoder_.SetQualityScaling(true);
5683 const int kWidth = 1280;
5684 const int kHeight = 720;
5685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115686 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085687 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5688 // Frame should not be dropped.
5689 WaitForEncodedFrame(1);
5690
5691 // Trigger QVGA "singlecast"
5692 // Update the config.
5693 VideoEncoderConfig video_encoder_config;
5694 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5695 &video_encoder_config);
5696 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5697 vp9_settings.numberOfSpatialLayers = 3;
5698 // Since only one layer is active - automatic resize should be enabled.
5699 vp9_settings.automaticResizeOn = true;
5700 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435701 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085702 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115703 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085704 video_encoder_config.content_type =
5705 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 14:22:115706 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085707 // which SVC layers are active.
5708 video_encoder_config.simulcast_layers.resize(3);
5709
5710 video_encoder_config.simulcast_layers[0].active = true;
5711 video_encoder_config.simulcast_layers[1].active = false;
5712 video_encoder_config.simulcast_layers[2].active = false;
5713
5714 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5715 kMaxPayloadLength);
5716 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5717
5718 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5719 // Frame should not be dropped.
5720 WaitForEncodedFrame(2);
5721
5722 // Trigger HD "singlecast"
5723 video_encoder_config.simulcast_layers[0].active = false;
5724 video_encoder_config.simulcast_layers[1].active = false;
5725 video_encoder_config.simulcast_layers[2].active = true;
5726
5727 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5728 kMaxPayloadLength);
5729 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5730
5731 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5732 // Frame should be dropped because of initial frame drop.
5733 ExpectDroppedFrame();
5734
5735 // Expect the sink_wants to specify a scaled frame.
5736 EXPECT_TRUE_WAIT(
5737 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5738 video_stream_encoder_->Stop();
5739}
5740
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475741TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055742 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5743 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5744 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5745 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5746 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5747 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5748 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5749 fake_encoder_.SetResolutionBitrateLimits(
5750 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5751
5752 VideoEncoderConfig video_encoder_config;
5753 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5754 &video_encoder_config);
5755 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5756 vp9_settings.numberOfSpatialLayers = 3;
5757 // Since only one layer is active - automatic resize should be enabled.
5758 vp9_settings.automaticResizeOn = true;
5759 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435760 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 08:20:055761 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115762 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 08:20:055763 video_encoder_config.content_type =
5764 VideoEncoderConfig::ContentType::kRealtimeVideo;
5765 // Simulcast layers are used to indicate which spatial layers are active.
5766 video_encoder_config.simulcast_layers.resize(3);
5767 video_encoder_config.simulcast_layers[0].active = false;
5768 video_encoder_config.simulcast_layers[1].active = true;
5769 video_encoder_config.simulcast_layers[2].active = false;
5770
5771 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5772 kMaxPayloadLength);
5773 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5774
5775 // The encoder bitrate limits for 360p should be used.
5776 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5777 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115778 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5779 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5780 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5781 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5782 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5783 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055784 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115785 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055786 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115787 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055788
5789 // The encoder bitrate limits for 270p should be used.
5790 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5791 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115792 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5793 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5794 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5795 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5796 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5797 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055798 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115799 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055800 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115801 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055802
5803 video_stream_encoder_->Stop();
5804}
5805
5806TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:515807 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5808 VideoEncoderConfig video_encoder_config;
5809 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5810 &video_encoder_config);
5811 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5812 vp9_settings.numberOfSpatialLayers = 3;
5813 // Since only one layer is active - automatic resize should be enabled.
5814 vp9_settings.automaticResizeOn = true;
5815 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435816 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 09:39:515817 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115818 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 09:39:515819 video_encoder_config.content_type =
5820 VideoEncoderConfig::ContentType::kRealtimeVideo;
5821 // Simulcast layers are used to indicate which spatial layers are active.
5822 video_encoder_config.simulcast_layers.resize(3);
5823 video_encoder_config.simulcast_layers[0].active = false;
5824 video_encoder_config.simulcast_layers[1].active = true;
5825 video_encoder_config.simulcast_layers[2].active = false;
5826
5827 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5828 kMaxPayloadLength);
5829 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5830
5831 // The default bitrate limits for 360p should be used.
5832 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195833 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5834 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515835 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5836 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115837 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5838 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5839 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5840 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5841 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5842 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515843 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115844 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515845 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115846 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515847
5848 // The default bitrate limits for 270p should be used.
5849 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 12:29:195850 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5851 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:515852 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5853 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115854 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5855 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5856 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5857 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5858 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5859 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515860 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115861 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515862 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115863 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515864
5865 video_stream_encoder_->Stop();
5866}
5867
5868TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 14:12:075869 webrtc::test::ScopedKeyValueConfig field_trials(
5870 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 09:39:515871 VideoEncoderConfig video_encoder_config;
5872 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5873 &video_encoder_config);
5874 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5875 vp9_settings.numberOfSpatialLayers = 3;
5876 // Since only one layer is active - automatic resize should be enabled.
5877 vp9_settings.automaticResizeOn = true;
5878 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435879 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 09:39:515880 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115881 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 09:39:515882 video_encoder_config.content_type =
5883 VideoEncoderConfig::ContentType::kRealtimeVideo;
5884 // Simulcast layers are used to indicate which spatial layers are active.
5885 video_encoder_config.simulcast_layers.resize(3);
5886 video_encoder_config.simulcast_layers[0].active = false;
5887 video_encoder_config.simulcast_layers[1].active = true;
5888 video_encoder_config.simulcast_layers[2].active = false;
5889
5890 // Reset encoder for field trials to take effect.
5891 ConfigureEncoder(video_encoder_config.Copy());
5892
5893 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5894 kMaxPayloadLength);
5895 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5896
5897 // The default bitrate limits for 360p should not be used.
5898 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195899 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5900 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515901 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5902 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115903 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5904 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5905 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5906 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5907 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5908 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515909 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115910 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515911
5912 video_stream_encoder_->Stop();
5913}
5914
5915TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5916 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5917 /*num_spatial_layers=*/1, /*screenshare=*/false);
5918
5919 // The default singlecast bitrate limits for 720p should not be used.
5920 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 12:29:195921 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5922 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 09:39:515923 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5924 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115925 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5926 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5927 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5928 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5929 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5930 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515931 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115932 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515933
5934 video_stream_encoder_->Stop();
5935}
5936
5937TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055938 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5939 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5940 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5941 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5942 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5943 fake_encoder_.SetResolutionBitrateLimits(
5944 {kEncoderLimits180p, kEncoderLimits720p});
5945
5946 VideoEncoderConfig video_encoder_config;
5947 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5948 &video_encoder_config);
5949 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5950 vp9_settings.numberOfSpatialLayers = 3;
5951 // Since only one layer is active - automatic resize should be enabled.
5952 vp9_settings.automaticResizeOn = true;
5953 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435954 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 08:20:055955 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115956 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 08:20:055957 video_encoder_config.content_type =
5958 VideoEncoderConfig::ContentType::kRealtimeVideo;
5959 // Simulcast layers are used to indicate which spatial layers are active.
5960 video_encoder_config.simulcast_layers.resize(3);
5961 video_encoder_config.simulcast_layers[0].active = true;
5962 video_encoder_config.simulcast_layers[1].active = false;
5963 video_encoder_config.simulcast_layers[2].active = false;
5964
5965 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5966 kMaxPayloadLength);
5967 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5968
5969 // Limits not applied on lowest stream, limits for 180p should not be used.
5970 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5971 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115972 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5973 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5974 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5975 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5976 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5977 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055978 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115979 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055980 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115981 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055982
5983 video_stream_encoder_->Stop();
5984}
5985
5986TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475987 InitialFrameDropActivatesWhenResolutionIncreases) {
5988 const int kWidth = 640;
5989 const int kHeight = 360;
5990
5991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115992 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475993 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5994 // Frame should not be dropped.
5995 WaitForEncodedFrame(1);
5996
5997 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115998 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475999 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6000 // Frame should not be dropped, bitrate not too low for frame.
6001 WaitForEncodedFrame(2);
6002
6003 // Incoming resolution increases.
6004 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6005 // Expect to drop this frame, bitrate too low for frame.
6006 ExpectDroppedFrame();
6007
6008 // Expect the sink_wants to specify a scaled frame.
6009 EXPECT_TRUE_WAIT(
6010 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6011 video_stream_encoder_->Stop();
6012}
6013
6014TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6015 const int kWidth = 640;
6016 const int kHeight = 360;
6017 // So that quality scaling doesn't happen by itself.
6018 fake_encoder_.SetQp(kQpHigh);
6019
6020 AdaptingFrameForwarder source(&time_controller_);
6021 source.set_adaptation_enabled(true);
6022 video_stream_encoder_->SetSource(
6023 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6024
6025 int timestamp = 1;
6026
6027 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116028 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:476029 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6030 WaitForEncodedFrame(timestamp);
6031 timestamp += 9000;
6032 // Long pause to disable all first BWE drop logic.
6033 AdvanceTime(TimeDelta::Millis(1000));
6034
6035 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116036 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:476037 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6038 // Not dropped frame, as initial frame drop is disabled by now.
6039 WaitForEncodedFrame(timestamp);
6040 timestamp += 9000;
6041 AdvanceTime(TimeDelta::Millis(100));
6042
6043 // Quality adaptation down.
6044 video_stream_encoder_->TriggerQualityLow();
6045
6046 // Adaptation has an effect.
6047 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6048 5000);
6049
6050 // Frame isn't dropped as initial frame dropper is disabled.
6051 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6052 WaitForEncodedFrame(timestamp);
6053 timestamp += 9000;
6054 AdvanceTime(TimeDelta::Millis(100));
6055
6056 // Quality adaptation up.
6057 video_stream_encoder_->TriggerQualityHigh();
6058
6059 // Adaptation has an effect.
6060 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6061 5000);
6062
6063 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6064 // Frame should not be dropped, as initial framedropper is off.
6065 WaitForEncodedFrame(timestamp);
6066
6067 video_stream_encoder_->Stop();
6068}
6069
Åsa Persson7f354f82021-02-04 14:52:156070TEST_F(VideoStreamEncoderTest,
6071 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6072 const int kMinStartBps360p = 222000;
6073 fake_encoder_.SetResolutionBitrateLimits(
6074 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6075 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6076 800000)});
6077
6078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6079 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6080 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6081 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6082 0, 0, 0);
6083 // Frame should not be dropped, bitrate not too low for frame.
6084 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6085 WaitForEncodedFrame(1);
6086
6087 // Incoming resolution increases, initial frame drop activates.
6088 // Frame should be dropped, link allocation too low for frame.
6089 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6090 ExpectDroppedFrame();
6091
6092 // Expect sink_wants to specify a scaled frame.
6093 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6094 5000);
6095 video_stream_encoder_->Stop();
6096}
6097
6098TEST_F(VideoStreamEncoderTest,
6099 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6100 const int kMinStartBps360p = 222000;
6101 fake_encoder_.SetResolutionBitrateLimits(
6102 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6103 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6104 800000)});
6105
6106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6107 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6108 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6109 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6110 0, 0, 0);
6111 // Frame should not be dropped, bitrate not too low for frame.
6112 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6113 WaitForEncodedFrame(1);
6114
6115 // Incoming resolution increases, initial frame drop activates.
6116 // Frame should be dropped, link allocation not too low for frame.
6117 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6118 WaitForEncodedFrame(2);
6119
6120 video_stream_encoder_->Stop();
6121}
6122
Åsa Perssone644a032019-11-08 14:56:006123TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 14:12:076124 webrtc::test::ScopedKeyValueConfig field_trials(
6125 field_trials_,
Åsa Persson06defc42021-09-10 13:28:486126 "WebRTC-Video-QualityRampupSettings/"
6127 "min_pixels:921600,min_duration_ms:2000/");
6128
6129 const int kWidth = 1280;
6130 const int kHeight = 720;
6131 const int kFps = 10;
6132 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 14:56:006133
6134 // Reset encoder for field trials to take effect.
6135 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 08:07:116136 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 09:34:326137 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 14:56:006138 ConfigureEncoder(std::move(config));
6139 fake_encoder_.SetQp(kQpLow);
6140
6141 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 12:31:236142 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 14:56:006143 source.set_adaptation_enabled(true);
6144 video_stream_encoder_->SetSource(&source,
6145 DegradationPreference::MAINTAIN_FRAMERATE);
6146
6147 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 08:07:116148 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 16:49:076149 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116150 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:006151
6152 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 13:28:486153 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 14:56:006154 int64_t timestamp_ms = kFrameIntervalMs;
6155 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6156 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:096157 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6158 5000);
Åsa Perssone644a032019-11-08 14:56:006159
6160 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 16:49:076161 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6162 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:006163
Artem Titovab30d722021-07-27 14:22:116164 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 13:28:486165 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 14:56:006166 for (size_t i = 1; i <= 10; i++) {
6167 timestamp_ms += kFrameIntervalMs;
6168 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6169 WaitForEncodedFrame(timestamp_ms);
6170 }
Åsa Persson06defc42021-09-10 13:28:486171
6172 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6173 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6174 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6175 timestamp_ms += kFrameIntervalMs;
6176 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6177 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 14:56:006178 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6179 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6180
Åsa Persson06defc42021-09-10 13:28:486181 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 14:56:006182 timestamp_ms += kFrameIntervalMs;
6183 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6184 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 16:49:076185 // The ramp-up code involves the adaptation queue, give it time to execute.
6186 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 11:12:126187 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 12:08:396188 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 14:56:006189
6190 // Frame should not be adapted.
6191 timestamp_ms += kFrameIntervalMs;
6192 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6193 WaitForEncodedFrame(kWidth, kHeight);
6194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6195
6196 video_stream_encoder_->Stop();
6197}
6198
mflodmancc3d4422017-08-03 15:27:516199TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 11:12:286200 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 14:12:076201 webrtc::test::ScopedKeyValueConfig field_trials(
6202 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 12:31:236203 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 11:12:286204 source.set_adaptation_enabled(true);
6205 video_stream_encoder_->SetSource(&source,
6206 DegradationPreference::MAINTAIN_FRAMERATE);
6207 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116208 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 11:12:286209 fake_encoder_.SetQp(kQpHigh + 1);
6210 const int kWidth = 1280;
6211 const int kHeight = 720;
6212 const int64_t kFrameIntervalMs = 100;
6213 int64_t timestamp_ms = kFrameIntervalMs;
6214 for (size_t i = 1; i <= 100; i++) {
6215 timestamp_ms += kFrameIntervalMs;
6216 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6217 WaitForEncodedFrame(timestamp_ms);
6218 }
6219 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6220 // for the first time.
6221 // TODO(eshr): We should avoid these waits by using threads with simulated
6222 // time.
6223 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6224 2000 * 2.5 * 2);
6225 timestamp_ms += kFrameIntervalMs;
6226 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6227 WaitForEncodedFrame(timestamp_ms);
6228 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6229 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6230 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6231
6232 // Disable Quality scaling by turning off scaler on the encoder and
6233 // reconfiguring.
6234 fake_encoder_.SetQualityScaling(false);
6235 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6236 kMaxPayloadLength);
6237 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 09:11:556238 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 11:12:286239 // Since we turned off the quality scaler, the adaptations made by it are
6240 // removed.
6241 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6242 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6243
6244 video_stream_encoder_->Stop();
6245}
6246
6247TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:316248 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6249 const int kTooSmallWidth = 10;
6250 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 16:49:076251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116252 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:316253
Taylor Brandstetter49fcc102018-05-16 21:20:416254 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:316255 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:516256 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416257 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 12:08:396258 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 08:47:316259 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6260
6261 // Trigger adapt down, too small frame, expect no change.
6262 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526263 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516264 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:396265 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:316266 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6267 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6268
mflodmancc3d4422017-08-03 15:27:516269 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:316270}
6271
mflodmancc3d4422017-08-03 15:27:516272TEST_F(VideoStreamEncoderTest,
6273 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226274 const int kTooSmallWidth = 10;
6275 const int kTooSmallHeight = 10;
6276 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 16:49:076277 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116278 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226279
Taylor Brandstetter49fcc102018-05-16 21:20:416280 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:226281 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:416282 video_stream_encoder_->SetSource(&source,
6283 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:396284 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:226285 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6286 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6287
6288 // Trigger adapt down, expect limited framerate.
6289 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526290 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516291 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396292 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226293 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6294 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6295 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6296
6297 // Trigger adapt down, too small frame, expect no change.
6298 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526299 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:516300 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396301 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226302 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6303 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6304 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6305
mflodmancc3d4422017-08-03 15:27:516306 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226307}
6308
mflodmancc3d4422017-08-03 15:27:516309TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 08:12:526310 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 16:49:076311 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116312 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 07:51:476313 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 08:12:526314 const int kFrameWidth = 1280;
6315 const int kFrameHeight = 720;
6316 video_source_.IncomingCapturedFrame(
6317 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526318 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 15:27:516319 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:526320}
6321
sprangb1ca0732017-02-01 16:38:126322// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 15:27:516323TEST_F(VideoStreamEncoderTest,
6324 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 16:49:076325 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116326 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 16:38:126327
6328 const int kFrameWidth = 1280;
6329 const int kFrameHeight = 720;
6330 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 15:27:516331 // requested by
6332 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 16:38:126333 video_source_.set_adaptation_enabled(true);
6334
6335 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196336 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526337 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126338
6339 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516340 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 16:38:126341 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196342 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526343 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 16:38:126344
asaperssonfab67072017-04-04 12:51:496345 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 10:24:336346 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 16:38:126347 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196348 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526349 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126350
mflodmancc3d4422017-08-03 15:27:516351 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 16:38:126352}
sprangfe627f32017-03-29 15:24:596353
mflodmancc3d4422017-08-03 15:27:516354TEST_F(VideoStreamEncoderTest,
6355 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-03 06:53:046356 const int kFrameWidth = 1280;
6357 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:046358
Henrik Boström381d1092020-05-12 16:49:076359 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116360 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516361 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416362 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046363 video_source_.set_adaptation_enabled(true);
6364
Tomas Gunnarsson612445e2020-09-21 12:31:236365 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046366
6367 video_source_.IncomingCapturedFrame(
6368 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526369 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046370
6371 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 15:27:516372 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046373
6374 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 14:06:526375 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046376 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046377 video_source_.IncomingCapturedFrame(
6378 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526379 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046380 }
6381
6382 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 15:27:516383 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046384 int num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526385 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046386 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046387 video_source_.IncomingCapturedFrame(
6388 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526389 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046390 ++num_frames_dropped;
6391 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566392 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046393 }
6394 }
6395
sprang4847ae62017-06-27 14:06:526396 // Add some slack to account for frames dropped by the frame dropper.
6397 const int kErrorMargin = 1;
6398 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046399 kErrorMargin);
6400
6401 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 15:27:516402 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046403 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 08:42:196404 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046405 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046406 video_source_.IncomingCapturedFrame(
6407 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526408 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046409 ++num_frames_dropped;
6410 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566411 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046412 }
6413 }
sprang4847ae62017-06-27 14:06:526414 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-03 06:53:046415 kErrorMargin);
6416
6417 // Go back up one step.
Henrik Boström91aa7322020-04-28 10:24:336418 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046419 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526420 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046421 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046422 video_source_.IncomingCapturedFrame(
6423 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526424 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046425 ++num_frames_dropped;
6426 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566427 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046428 }
6429 }
sprang4847ae62017-06-27 14:06:526430 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046431 kErrorMargin);
6432
6433 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 10:24:336434 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046435 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526436 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046437 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046438 video_source_.IncomingCapturedFrame(
6439 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526440 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046441 ++num_frames_dropped;
6442 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566443 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046444 }
6445 }
6446 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6447
mflodmancc3d4422017-08-03 15:27:516448 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046449}
6450
mflodmancc3d4422017-08-03 15:27:516451TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-03 06:53:046452 const int kFramerateFps = 5;
6453 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-03 06:53:046454 const int kFrameWidth = 1280;
6455 const int kFrameHeight = 720;
6456
sprang4847ae62017-06-27 14:06:526457 // Reconfigure encoder with two temporal layers and screensharing, which will
6458 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 07:51:476459 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 14:06:526460
Henrik Boström381d1092020-05-12 16:49:076461 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116462 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516463 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416464 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046465 video_source_.set_adaptation_enabled(true);
6466
Tomas Gunnarsson612445e2020-09-21 12:31:236467 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046468
6469 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-09 01:04:296470 rtc::VideoSinkWants last_wants;
6471 do {
6472 last_wants = video_source_.sink_wants();
6473
sprangc5d62e22017-04-03 06:53:046474 // Insert frames to get a new fps estimate...
6475 for (int j = 0; j < kFramerateFps; ++j) {
6476 video_source_.IncomingCapturedFrame(
6477 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-09 01:04:296478 if (video_source_.last_sent_width()) {
6479 sink_.WaitForEncodedFrame(timestamp_ms);
6480 }
sprangc5d62e22017-04-03 06:53:046481 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:236482 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-03 06:53:046483 }
6484 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 15:27:516485 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:296486 } while (video_source_.sink_wants().max_framerate_fps <
6487 last_wants.max_framerate_fps);
sprangc5d62e22017-04-03 06:53:046488
Evan Shrubsole5cd7eb82020-05-25 12:08:396489 EXPECT_THAT(video_source_.sink_wants(),
6490 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-14 06:25:226491
mflodmancc3d4422017-08-03 15:27:516492 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046493}
asaperssonf7e294d2017-06-14 06:25:226494
mflodmancc3d4422017-08-03 15:27:516495TEST_F(VideoStreamEncoderTest,
6496 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226497 const int kWidth = 1280;
6498 const int kHeight = 720;
6499 const int64_t kFrameIntervalMs = 150;
6500 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116502 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226503
Taylor Brandstetter49fcc102018-05-16 21:20:416504 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236505 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226506 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416507 video_stream_encoder_->SetSource(&source,
6508 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226509 timestamp_ms += kFrameIntervalMs;
6510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526511 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396512 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226513 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6514 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6515 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6516
6517 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516518 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226519 timestamp_ms += kFrameIntervalMs;
6520 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526521 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396522 EXPECT_THAT(source.sink_wants(),
6523 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6525 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6526 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6527
6528 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516529 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226530 timestamp_ms += kFrameIntervalMs;
6531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526532 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546533 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6535 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6536 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6537
6538 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516539 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226540 timestamp_ms += kFrameIntervalMs;
6541 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526542 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546543 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226544 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6545 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6546 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6547
6548 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516549 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226550 timestamp_ms += kFrameIntervalMs;
6551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526552 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546553 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6556 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6557
6558 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516559 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226560 timestamp_ms += kFrameIntervalMs;
6561 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526562 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546563 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6566 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6567
6568 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516569 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226570 timestamp_ms += kFrameIntervalMs;
6571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526572 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546573 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6576 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6577
6578 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 15:27:516579 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226580 timestamp_ms += kFrameIntervalMs;
6581 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526582 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546583 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226584 rtc::VideoSinkWants last_wants = source.sink_wants();
6585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6587 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6588
6589 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 15:27:516590 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226591 timestamp_ms += kFrameIntervalMs;
6592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526593 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546594 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-14 06:25:226595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6597 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6598
Åsa Perssonf5f7e8e2021-06-09 20:55:386599 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516600 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226601 timestamp_ms += kFrameIntervalMs;
6602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526603 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546604 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226605 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6606 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6607 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6608
6609 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516610 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226611 timestamp_ms += kFrameIntervalMs;
6612 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526613 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546614 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226615 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6617 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6618
6619 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516620 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226621 timestamp_ms += kFrameIntervalMs;
6622 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526623 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546624 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226625 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6627 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6628
6629 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516630 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226631 timestamp_ms += kFrameIntervalMs;
6632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526633 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546634 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226635 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6637 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6638
6639 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516640 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226641 timestamp_ms += kFrameIntervalMs;
6642 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526643 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546644 EXPECT_THAT(source.sink_wants(), FpsMax());
6645 EXPECT_EQ(source.sink_wants().max_pixel_count,
6646 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-14 06:25:226647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6649 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6650
6651 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516652 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226653 timestamp_ms += kFrameIntervalMs;
6654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526655 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546656 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226657 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6658 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6659 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6660
Åsa Persson30ab0152019-08-27 10:22:336661 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516662 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226663 timestamp_ms += kFrameIntervalMs;
6664 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526665 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546666 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396667 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226668 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6669 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6670 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6671
6672 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516673 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396674 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226675 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6676
mflodmancc3d4422017-08-03 15:27:516677 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226678}
6679
mflodmancc3d4422017-08-03 15:27:516680TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-14 06:25:226681 const int kWidth = 1280;
6682 const int kHeight = 720;
6683 const int64_t kFrameIntervalMs = 150;
6684 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116686 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226687
Taylor Brandstetter49fcc102018-05-16 21:20:416688 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236689 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226690 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416691 video_stream_encoder_->SetSource(&source,
6692 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226693 timestamp_ms += kFrameIntervalMs;
6694 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526695 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396696 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226697 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6698 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6699 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6700 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6701 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6702 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6703
6704 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516705 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226706 timestamp_ms += kFrameIntervalMs;
6707 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526708 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396709 EXPECT_THAT(source.sink_wants(),
6710 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226711 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6712 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6713 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6714 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6715 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6716 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6717
6718 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516719 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226720 timestamp_ms += kFrameIntervalMs;
6721 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526722 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546723 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226724 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6725 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6726 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6727 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6728 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6729 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6730
6731 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516732 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226733 timestamp_ms += kFrameIntervalMs;
6734 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526735 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546736 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 08:45:296737 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-14 06:25:226738 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6739 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6740 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6741 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6742 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6743
Evan Shrubsole64469032020-06-11 08:45:296744 // Trigger cpu adapt up, expect no change since QP is most limited.
6745 {
6746 // Store current sink wants since we expect no change and if there is no
6747 // change then last_wants() is not updated.
6748 auto previous_sink_wants = source.sink_wants();
6749 video_stream_encoder_->TriggerCpuUnderuse();
6750 timestamp_ms += kFrameIntervalMs;
6751 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6752 WaitForEncodedFrame(timestamp_ms);
6753 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6754 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6755 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6756 }
6757
6758 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6759 video_stream_encoder_->TriggerQualityHigh();
6760 timestamp_ms += kFrameIntervalMs;
6761 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6762 WaitForEncodedFrame(timestamp_ms);
6763 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6764 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6765 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6766 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6767 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6768 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6769 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6770
6771 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6772 // expect increased resolution (960x540@30fps).
6773 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336774 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226775 timestamp_ms += kFrameIntervalMs;
6776 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526777 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 08:45:296778 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226779 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6780 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6781 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6782 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6783 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296784 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226785
Evan Shrubsole64469032020-06-11 08:45:296786 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6787 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516788 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336789 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226790 timestamp_ms += kFrameIntervalMs;
6791 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526792 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546793 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396794 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226795 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6797 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6798 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6799 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296800 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226801
6802 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516803 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396804 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226805 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296806 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226807
mflodmancc3d4422017-08-03 15:27:516808 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226809}
6810
mflodmancc3d4422017-08-03 15:27:516811TEST_F(VideoStreamEncoderTest,
6812 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-14 06:25:226813 const int kWidth = 640;
6814 const int kHeight = 360;
6815 const int kFpsLimit = 15;
6816 const int64_t kFrameIntervalMs = 150;
6817 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076818 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116819 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226820
Taylor Brandstetter49fcc102018-05-16 21:20:416821 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236822 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226823 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416824 video_stream_encoder_->SetSource(&source,
6825 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226826 timestamp_ms += kFrameIntervalMs;
6827 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526828 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396829 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226830 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6831 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6832 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6833 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6834 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6835 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6836
6837 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516838 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226839 timestamp_ms += kFrameIntervalMs;
6840 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526841 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396842 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226843 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6844 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6845 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6846 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6847 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6848 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6849
6850 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516851 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226852 timestamp_ms += kFrameIntervalMs;
6853 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526854 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546855 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226856 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296857 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:226858 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6859 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6861 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6862
Evan Shrubsole64469032020-06-11 08:45:296863 // Trigger cpu adapt up, expect no change because quality is most limited.
6864 {
6865 auto previous_sink_wants = source.sink_wants();
6866 // Store current sink wants since we expect no change ind if there is no
6867 // change then last__wants() is not updated.
6868 video_stream_encoder_->TriggerCpuUnderuse();
6869 timestamp_ms += kFrameIntervalMs;
6870 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6871 WaitForEncodedFrame(timestamp_ms);
6872 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6873 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6874 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6875 }
6876
6877 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6878 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226879 timestamp_ms += kFrameIntervalMs;
6880 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526881 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546882 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6884 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6885 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296886 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6887 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6888 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226889
Evan Shrubsole64469032020-06-11 08:45:296890 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516891 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 08:45:296892 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226893 timestamp_ms += kFrameIntervalMs;
6894 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526895 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396896 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226897 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6898 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6899 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6900 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6901 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296902 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226903
6904 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516905 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396906 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226907 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296908 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226909
mflodmancc3d4422017-08-03 15:27:516910 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226911}
6912
mflodmancc3d4422017-08-03 15:27:516913TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 13:53:486914 const int kFrameWidth = 1920;
6915 const int kFrameHeight = 1080;
6916 // 3/4 of 1920.
6917 const int kAdaptedFrameWidth = 1440;
6918 // 3/4 of 1080 rounded down to multiple of 4.
6919 const int kAdaptedFrameHeight = 808;
6920 const int kFramerate = 24;
6921
Henrik Boström381d1092020-05-12 16:49:076922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116923 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 13:53:486924 // Trigger reconfigure encoder (without resetting the entire instance).
6925 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 10:57:586926 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6927 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:116928 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 13:53:486929 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:436930 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 15:27:516931 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:476932 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:516933 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 13:53:486934
6935 video_source_.set_adaptation_enabled(true);
6936
6937 video_source_.IncomingCapturedFrame(
6938 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526939 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 13:53:486940
6941 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516942 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 13:53:486943 video_source_.IncomingCapturedFrame(
6944 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526945 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 13:53:486946
mflodmancc3d4422017-08-03 15:27:516947 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 13:53:486948}
6949
mflodmancc3d4422017-08-03 15:27:516950TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 14:06:526951 const int kFrameWidth = 1280;
6952 const int kFrameHeight = 720;
6953 const int kLowFps = 2;
6954 const int kHighFps = 30;
6955
Henrik Boström381d1092020-05-12 16:49:076956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116957 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:526958
Tomas Gunnarsson612445e2020-09-21 12:31:236959 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:526960 max_framerate_ = kLowFps;
6961
6962 // Insert 2 seconds of 2fps video.
6963 for (int i = 0; i < kLowFps * 2; ++i) {
6964 video_source_.IncomingCapturedFrame(
6965 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6966 WaitForEncodedFrame(timestamp_ms);
6967 timestamp_ms += 1000 / kLowFps;
6968 }
6969
6970 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 16:49:076971 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116972 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:526973 video_source_.IncomingCapturedFrame(
6974 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6975 WaitForEncodedFrame(timestamp_ms);
6976 timestamp_ms += 1000 / kLowFps;
6977
6978 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6979
6980 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 08:48:486981 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 14:06:526982 const int kFrameIntervalMs = 1000 / kHighFps;
6983 max_framerate_ = kHighFps;
6984 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6985 video_source_.IncomingCapturedFrame(
6986 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6987 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6988 // be dropped if the encoder hans't been updated with the new higher target
6989 // framerate yet, causing it to overshoot the target bitrate and then
6990 // suffering the wrath of the media optimizer.
6991 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6992 timestamp_ms += kFrameIntervalMs;
6993 }
6994
6995 // Don expect correct measurement just yet, but it should be higher than
6996 // before.
6997 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6998
mflodmancc3d4422017-08-03 15:27:516999 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:527000}
7001
mflodmancc3d4422017-08-03 15:27:517002TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 14:06:527003 const int kFrameWidth = 1280;
7004 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 13:09:057005 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:037006 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:057007 kVideoBitrateAllocation);
sprang4847ae62017-06-27 14:06:527008
Henrik Boström381d1092020-05-12 16:49:077009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117010 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:517011 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 14:06:527012
7013 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 12:31:237014 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:527015 video_source_.IncomingCapturedFrame(
7016 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7017 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 13:09:057018 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:527019
7020 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 16:49:077021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117022 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 14:06:527023
7024 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 08:48:487025 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:237026 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 14:06:527027
Per Kjellanderdcef6412020-10-07 13:09:057028 // No more allocations has been made.
sprang4847ae62017-06-27 14:06:527029 video_source_.IncomingCapturedFrame(
7030 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7031 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 13:09:057032 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:527033
mflodmancc3d4422017-08-03 15:27:517034 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:527035}
ilnik6b826ef2017-06-16 13:53:487036
Niels Möller4db138e2018-04-19 07:04:137037TEST_F(VideoStreamEncoderTest,
7038 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7039 const int kFrameWidth = 1280;
7040 const int kFrameHeight = 720;
7041 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 16:49:077042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117043 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:137044 video_source_.IncomingCapturedFrame(
7045 CreateFrame(1, kFrameWidth, kFrameHeight));
7046 WaitForEncodedFrame(1);
7047 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7048 .low_encode_usage_threshold_percent,
7049 default_options.low_encode_usage_threshold_percent);
7050 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7051 .high_encode_usage_threshold_percent,
7052 default_options.high_encode_usage_threshold_percent);
7053 video_stream_encoder_->Stop();
7054}
7055
7056TEST_F(VideoStreamEncoderTest,
7057 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7058 const int kFrameWidth = 1280;
7059 const int kFrameHeight = 720;
7060 CpuOveruseOptions hardware_options;
7061 hardware_options.low_encode_usage_threshold_percent = 150;
7062 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 11:42:187063 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 07:04:137064
Henrik Boström381d1092020-05-12 16:49:077065 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117066 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:137067 video_source_.IncomingCapturedFrame(
7068 CreateFrame(1, kFrameWidth, kFrameHeight));
7069 WaitForEncodedFrame(1);
7070 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7071 .low_encode_usage_threshold_percent,
7072 hardware_options.low_encode_usage_threshold_percent);
7073 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7074 .high_encode_usage_threshold_percent,
7075 hardware_options.high_encode_usage_threshold_percent);
7076 video_stream_encoder_->Stop();
7077}
7078
Jakob Ivarsson461b1d92021-01-22 15:27:437079TEST_F(VideoStreamEncoderTest,
7080 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7081 const int kFrameWidth = 1280;
7082 const int kFrameHeight = 720;
7083
7084 const CpuOveruseOptions default_options;
7085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117086 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 15:27:437087 video_source_.IncomingCapturedFrame(
7088 CreateFrame(1, kFrameWidth, kFrameHeight));
7089 WaitForEncodedFrame(1);
7090 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7091 .low_encode_usage_threshold_percent,
7092 default_options.low_encode_usage_threshold_percent);
7093 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7094 .high_encode_usage_threshold_percent,
7095 default_options.high_encode_usage_threshold_percent);
7096
7097 CpuOveruseOptions hardware_options;
7098 hardware_options.low_encode_usage_threshold_percent = 150;
7099 hardware_options.high_encode_usage_threshold_percent = 200;
7100 fake_encoder_.SetIsHardwareAccelerated(true);
7101
7102 video_source_.IncomingCapturedFrame(
7103 CreateFrame(2, kFrameWidth, kFrameHeight));
7104 WaitForEncodedFrame(2);
7105
7106 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7107 .low_encode_usage_threshold_percent,
7108 hardware_options.low_encode_usage_threshold_percent);
7109 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7110 .high_encode_usage_threshold_percent,
7111 hardware_options.high_encode_usage_threshold_percent);
7112
7113 video_stream_encoder_->Stop();
7114}
7115
Niels Möller6bb5ab92019-01-11 10:11:107116TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7117 const int kFrameWidth = 320;
7118 const int kFrameHeight = 240;
7119 const int kFps = 30;
Asa Persson606d3cb2021-10-04 08:07:117120 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 10:11:107121 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7122
Henrik Boström381d1092020-05-12 16:49:077123 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117124 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107125
Tomas Gunnarsson612445e2020-09-21 12:31:237126 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:107127 max_framerate_ = kFps;
7128
7129 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7130 fake_encoder_.SimulateOvershoot(1.0);
7131 int num_dropped = 0;
7132 for (int i = 0; i < kNumFramesInRun; ++i) {
7133 video_source_.IncomingCapturedFrame(
7134 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7135 // Wait up to two frame durations for a frame to arrive.
7136 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7137 ++num_dropped;
7138 }
7139 timestamp_ms += 1000 / kFps;
7140 }
7141
Erik Språnga8d48ab2019-02-08 13:17:407142 // Framerate should be measured to be near the expected target rate.
7143 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7144
7145 // Frame drops should be within 5% of expected 0%.
7146 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:107147
7148 // Make encoder produce frames at double the expected bitrate during 3 seconds
7149 // of video, verify number of drops. Rate needs to be slightly changed in
7150 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 15:20:177151 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 15:44:427152 const RateControlSettings trials =
7153 RateControlSettings::ParseFromFieldTrials();
7154 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 15:20:177155 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 15:44:427156 // frame dropping since the adjuter will try to just lower the target
7157 // bitrate rather than drop frames. If network headroom can be used, it
7158 // doesn't push back as hard so we don't need quite as much overshoot.
7159 // These numbers are unfortunately a bit magical but there's not trivial
7160 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 10:10:097161 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 15:20:177162 }
7163 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 16:49:077164 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117165 kTargetBitrate + DataRate::KilobitsPerSec(1),
7166 kTargetBitrate + DataRate::KilobitsPerSec(1),
7167 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107168 num_dropped = 0;
7169 for (int i = 0; i < kNumFramesInRun; ++i) {
7170 video_source_.IncomingCapturedFrame(
7171 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7172 // Wait up to two frame durations for a frame to arrive.
7173 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7174 ++num_dropped;
7175 }
7176 timestamp_ms += 1000 / kFps;
7177 }
7178
Henrik Boström381d1092020-05-12 16:49:077179 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117180 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 13:17:407181
7182 // Target framerate should be still be near the expected target, despite
7183 // the frame drops.
7184 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7185
7186 // Frame drops should be within 5% of expected 50%.
7187 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:107188
7189 video_stream_encoder_->Stop();
7190}
7191
7192TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7193 const int kFrameWidth = 320;
7194 const int kFrameHeight = 240;
7195 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 08:07:117196 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 10:11:107197
7198 ASSERT_GT(max_framerate_, kActualInputFps);
7199
Tomas Gunnarsson612445e2020-09-21 12:31:237200 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:107201 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 16:49:077202 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117203 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107204
7205 // Insert 3 seconds of video, with an input fps lower than configured max.
7206 for (int i = 0; i < kActualInputFps * 3; ++i) {
7207 video_source_.IncomingCapturedFrame(
7208 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7209 // Wait up to two frame durations for a frame to arrive.
7210 WaitForEncodedFrame(timestamp_ms);
7211 timestamp_ms += 1000 / kActualInputFps;
7212 }
7213
7214 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7215
7216 video_stream_encoder_->Stop();
7217}
7218
Markus Handell9a478b52021-11-18 15:07:017219TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267220 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 15:07:017221 test::FrameForwarder source;
7222 video_stream_encoder_->SetSource(&source,
7223 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 16:49:077224 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117225 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267226
Markus Handell9a478b52021-11-18 15:07:017227 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267228 WaitForEncodedFrame(1);
7229 // On the very first frame full update should be forced.
7230 rect = fake_encoder_.GetLastUpdateRect();
7231 EXPECT_EQ(rect.offset_x, 0);
7232 EXPECT_EQ(rect.offset_y, 0);
7233 EXPECT_EQ(rect.height, codec_height_);
7234 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 15:07:017235 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7236 // scheduled for processing during encoder queue processing of frame 2.
7237 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7238 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267239 WaitForEncodedFrame(3);
7240 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7241 rect = fake_encoder_.GetLastUpdateRect();
7242 EXPECT_EQ(rect.offset_x, 1);
7243 EXPECT_EQ(rect.offset_y, 0);
7244 EXPECT_EQ(rect.width, 10);
7245 EXPECT_EQ(rect.height, 1);
7246
Markus Handell9a478b52021-11-18 15:07:017247 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267248 WaitForEncodedFrame(4);
7249 // Previous frame was encoded, so no accumulation should happen.
7250 rect = fake_encoder_.GetLastUpdateRect();
7251 EXPECT_EQ(rect.offset_x, 0);
7252 EXPECT_EQ(rect.offset_y, 0);
7253 EXPECT_EQ(rect.width, 1);
7254 EXPECT_EQ(rect.height, 1);
7255
7256 video_stream_encoder_->Stop();
7257}
7258
Erik Språngd7329ca2019-02-21 20:19:537259TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 16:49:077260 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117261 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537262
7263 // First frame is always keyframe.
7264 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7265 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 14:43:587266 EXPECT_THAT(
7267 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127268 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537269
7270 // Insert delta frame.
7271 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7272 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 14:43:587273 EXPECT_THAT(
7274 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127275 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537276
7277 // Request next frame be a key-frame.
7278 video_stream_encoder_->SendKeyFrame();
7279 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7280 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 14:43:587281 EXPECT_THAT(
7282 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127283 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537284
7285 video_stream_encoder_->Stop();
7286}
7287
7288TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7289 // Setup simulcast with three streams.
7290 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117292 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7293 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537294 // Wait for all three layers before triggering event.
7295 sink_.SetNumExpectedLayers(3);
7296
7297 // First frame is always keyframe.
7298 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7299 WaitForEncodedFrame(1);
7300 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127301 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7302 VideoFrameType::kVideoFrameKey,
7303 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537304
7305 // Insert delta frame.
7306 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7307 WaitForEncodedFrame(2);
7308 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127309 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7310 VideoFrameType::kVideoFrameDelta,
7311 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537312
7313 // Request next frame be a key-frame.
7314 // Only first stream is configured to produce key-frame.
7315 video_stream_encoder_->SendKeyFrame();
7316 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7317 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 11:45:397318
7319 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7320 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 20:19:537321 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127322 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 11:45:397323 VideoFrameType::kVideoFrameKey,
7324 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537325
7326 video_stream_encoder_->Stop();
7327}
7328
Mirta Dvornicic28f0eb22019-05-28 14:30:167329TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237330 // SPS contains VUI with restrictions on the maximum number of reordered
7331 // pictures, there is no need to rewrite the bitstream to enable faster
7332 // decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167333 ResetEncoder("H264", 1, 1, 1, false);
7334
Mirta Dvornicic97910da2020-07-14 13:29:237335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117336 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 13:29:237337 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167338
Mirta Dvornicic97910da2020-07-14 13:29:237339 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 08:07:117340 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167341
Mirta Dvornicic97910da2020-07-14 13:29:237342 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7343 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167344
7345 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 08:07:117346 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167347
7348 video_stream_encoder_->Stop();
7349}
7350
7351TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237352 // SPS does not contain VUI, the bitstream is will be rewritten with added
7353 // VUI with restrictions on the maximum number of reordered pictures to
7354 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167355 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7356 0x00, 0x00, 0x03, 0x03, 0xF4,
7357 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 14:30:167358 ResetEncoder("H264", 1, 1, 1, false);
7359
Mirta Dvornicic97910da2020-07-14 13:29:237360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117361 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 13:29:237362 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167363
Mirta Dvornicic97910da2020-07-14 13:29:237364 fake_encoder_.SetEncodedImageData(
7365 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167366
Mirta Dvornicic97910da2020-07-14 13:29:237367 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7368 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167369
7370 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 08:07:117371 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167372
7373 video_stream_encoder_->Stop();
7374}
7375
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147376TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7377 const int kFrameWidth = 1280;
7378 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 08:07:117379 const DataRate kTargetBitrate =
7380 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147381
Henrik Boström381d1092020-05-12 16:49:077382 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117383 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147384 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7385
7386 // Insert a first video frame. It should be dropped because of downscale in
7387 // resolution.
Tomas Gunnarsson612445e2020-09-21 12:31:237388 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147389 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7390 frame.set_rotation(kVideoRotation_270);
7391 video_source_.IncomingCapturedFrame(frame);
7392
7393 ExpectDroppedFrame();
7394
7395 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237396 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147397 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7398 frame.set_rotation(kVideoRotation_90);
7399 video_source_.IncomingCapturedFrame(frame);
7400
7401 WaitForEncodedFrame(timestamp_ms);
7402 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7403
7404 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237405 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147406 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7407 frame.set_rotation(kVideoRotation_180);
7408 video_source_.IncomingCapturedFrame(frame);
7409
7410 WaitForEncodedFrame(timestamp_ms);
7411 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7412
7413 video_stream_encoder_->Stop();
7414}
7415
Erik Språng5056af02019-09-02 13:53:117416TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7417 const int kFrameWidth = 320;
7418 const int kFrameHeight = 180;
7419
7420 // Initial rate.
Henrik Boström381d1092020-05-12 16:49:077421 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077422 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7423 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7424 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 13:53:117425 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487426 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327427 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117428
7429 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237430 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 13:53:117431 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7432 frame.set_rotation(kVideoRotation_270);
7433 video_source_.IncomingCapturedFrame(frame);
7434 WaitForEncodedFrame(timestamp_ms);
7435
7436 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 08:07:117437 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 17:46:077438 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7439 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 16:49:077440 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 13:53:117441 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 11:36:557442 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 13:53:117443 /*link_allocation=*/target_rate,
7444 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487445 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327446 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117447 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7448
7449 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7450 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7451 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 17:46:077452 DataRate allocation_sum =
7453 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 13:53:117454 EXPECT_EQ(min_rate, allocation_sum);
7455 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7456
7457 video_stream_encoder_->Stop();
7458}
7459
Rasmus Brandt5cad55b2019-12-19 08:47:117460TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 16:49:077461 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117462 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 10:50:237463 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 12:31:237464 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 10:50:237465 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7466 WaitForEncodedFrame(1);
7467
7468 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7469 ASSERT_TRUE(prev_rate_settings.has_value());
7470 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7471 kDefaultFramerate);
7472
7473 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7474 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7475 timestamp_ms += 1000 / kDefaultFramerate;
7476 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7477 WaitForEncodedFrame(timestamp_ms);
7478 }
7479 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7480 kDefaultFramerate);
7481 // Capture larger frame to trigger a reconfigure.
7482 codec_height_ *= 2;
7483 codec_width_ *= 2;
7484 timestamp_ms += 1000 / kDefaultFramerate;
7485 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7486 WaitForEncodedFrame(timestamp_ms);
7487
7488 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7489 auto current_rate_settings =
7490 fake_encoder_.GetAndResetLastRateControlSettings();
7491 // Ensure we have actually reconfigured twice
7492 // The rate settings should have been set again even though
7493 // they haven't changed.
7494 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 07:55:037495 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 10:50:237496
7497 video_stream_encoder_->Stop();
7498}
7499
philipeld9cc8c02019-09-16 12:53:407500struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 17:17:517501 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 17:17:517502 MOCK_METHOD(void,
7503 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 10:35:047504 (const webrtc::SdpVideoFormat& format,
7505 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 17:17:517506 (override));
philipeld9cc8c02019-09-16 12:53:407507};
7508
philipel9b058032020-02-10 10:30:007509TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7510 constexpr int kDontCare = 100;
7511 StrictMock<MockEncoderSelector> encoder_selector;
7512 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7513 &fake_encoder_, &encoder_selector);
7514 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7515
7516 // Reset encoder for new configuration to take effect.
7517 ConfigureEncoder(video_encoder_config_.Copy());
7518
Sergey Silkine1cd3ad2022-01-21 10:35:047519 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 10:30:007520
7521 video_source_.IncomingCapturedFrame(
7522 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 09:11:557523 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 10:30:007524 video_stream_encoder_->Stop();
7525
Sergey Silkine1cd3ad2022-01-21 10:35:047526 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 10:30:007527 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 14:22:117528 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7529 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 10:30:007530 video_stream_encoder_.reset();
7531}
7532
7533TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7534 constexpr int kDontCare = 100;
7535
7536 NiceMock<MockEncoderSelector> encoder_selector;
7537 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7538 video_send_config_.encoder_settings.encoder_switch_request_callback =
7539 &switch_callback;
7540 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7541 &fake_encoder_, &encoder_selector);
7542 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7543
7544 // Reset encoder for new configuration to take effect.
7545 ConfigureEncoder(video_encoder_config_.Copy());
7546
Sergey Silkine1cd3ad2022-01-21 10:35:047547 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 10:30:007548 .WillByDefault(Return(SdpVideoFormat("AV1")));
7549 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 10:35:047550 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7551 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 10:30:007552
Henrik Boström381d1092020-05-12 16:49:077553 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077554 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7555 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7556 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 10:30:007557 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487558 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 10:30:007559 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 09:11:557560 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 10:30:007561
7562 video_stream_encoder_->Stop();
7563}
7564
7565TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7566 constexpr int kSufficientBitrateToNotDrop = 1000;
7567 constexpr int kDontCare = 100;
7568
7569 NiceMock<MockVideoEncoder> video_encoder;
7570 NiceMock<MockEncoderSelector> encoder_selector;
7571 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7572 video_send_config_.encoder_settings.encoder_switch_request_callback =
7573 &switch_callback;
7574 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7575 &video_encoder, &encoder_selector);
7576 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7577
7578 // Reset encoder for new configuration to take effect.
7579 ConfigureEncoder(video_encoder_config_.Copy());
7580
7581 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7582 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7583 // not fail.
Henrik Boström381d1092020-05-12 16:49:077584 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077585 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7586 /*stable_target_bitrate=*/
7587 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7588 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 10:30:007589 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487590 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 10:30:007591 /*cwnd_reduce_ratio=*/0);
7592
Sergey Silkine1cd3ad2022-01-21 10:35:047593 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 10:30:007594 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 10:35:047595 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 10:30:007596 .WillByDefault(Return(SdpVideoFormat("AV2")));
7597
7598 rtc::Event encode_attempted;
7599 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 10:35:047600 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7601 /*allow_default_fallback=*/true))
7602 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 10:30:007603
7604 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7605 encode_attempted.Wait(3000);
7606
Markus Handell28c71802021-11-08 09:11:557607 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 13:56:427608
philipel9b058032020-02-10 10:30:007609 video_stream_encoder_->Stop();
7610
Sergey Silkine1cd3ad2022-01-21 10:35:047611 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7612 // to it's factory, so in order for the encoder instance in the
7613 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7614 // reset the `video_stream_encoder_` here.
7615 video_stream_encoder_.reset();
7616}
7617
7618TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7619 NiceMock<MockVideoEncoder> video_encoder;
7620 NiceMock<MockEncoderSelector> encoder_selector;
7621 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7622 video_send_config_.encoder_settings.encoder_switch_request_callback =
7623 &switch_callback;
7624 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7625 &video_encoder, &encoder_selector);
7626 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7627
7628 // Reset encoder for new configuration to take effect.
7629 ConfigureEncoder(video_encoder_config_.Copy());
7630
7631 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7632 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7633 /*round_trip_time_ms=*/0,
7634 /*cwnd_reduce_ratio=*/0);
7635 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7636
7637 ON_CALL(video_encoder, InitEncode(_, _))
7638 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7639 ON_CALL(encoder_selector, OnEncoderBroken)
7640 .WillByDefault(Return(SdpVideoFormat("AV2")));
7641
7642 rtc::Event encode_attempted;
7643 EXPECT_CALL(switch_callback,
7644 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7645 /*allow_default_fallback=*/true))
7646 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7647
7648 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7649 encode_attempted.Wait(3000);
7650
7651 AdvanceTime(TimeDelta::Zero());
7652
7653 video_stream_encoder_->Stop();
7654
7655 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7656 // to it's factory, so in order for the encoder instance in the
7657 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7658 // reset the `video_stream_encoder_` here.
7659 video_stream_encoder_.reset();
7660}
7661
7662TEST_F(VideoStreamEncoderTest,
7663 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7664 NiceMock<MockVideoEncoder> video_encoder;
7665 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7666 video_send_config_.encoder_settings.encoder_switch_request_callback =
7667 &switch_callback;
7668 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7669 &video_encoder, /*encoder_selector=*/nullptr);
7670 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7671
7672 // Reset encoder for new configuration to take effect.
7673 ConfigureEncoder(video_encoder_config_.Copy());
7674
7675 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7676 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7677 /*round_trip_time_ms=*/0,
7678 /*cwnd_reduce_ratio=*/0);
7679 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7680
7681 ON_CALL(video_encoder, InitEncode(_, _))
7682 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7683
7684 rtc::Event encode_attempted;
7685 EXPECT_CALL(switch_callback,
7686 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7687 /*allow_default_fallback=*/true))
7688 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7689
7690 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7691 encode_attempted.Wait(3000);
7692
7693 AdvanceTime(TimeDelta::Zero());
7694
7695 video_stream_encoder_->Stop();
7696
7697 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 10:30:007698 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 14:22:117699 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7700 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 10:30:007701 video_stream_encoder_.reset();
7702}
7703
Evan Shrubsole7c079f62019-09-26 07:55:037704TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117705 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037706 const int kFrameWidth = 320;
7707 const int kFrameHeight = 180;
7708
7709 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077710 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037712 /*target_bitrate=*/rate,
7713 /*stable_target_bitrate=*/rate,
7714 /*link_allocation=*/rate,
7715 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487716 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327717 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037718
7719 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237720 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037721 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7722 frame.set_rotation(kVideoRotation_270);
7723 video_source_.IncomingCapturedFrame(frame);
7724 WaitForEncodedFrame(timestamp_ms);
7725 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7726
7727 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 17:46:077728 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077729 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037730 /*target_bitrate=*/new_stable_rate,
7731 /*stable_target_bitrate=*/new_stable_rate,
7732 /*link_allocation=*/rate,
7733 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487734 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327735 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037736 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7737 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7738 video_stream_encoder_->Stop();
7739}
7740
7741TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117742 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037743 const int kFrameWidth = 320;
7744 const int kFrameHeight = 180;
7745
7746 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077747 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037749 /*target_bitrate=*/rate,
7750 /*stable_target_bitrate=*/rate,
7751 /*link_allocation=*/rate,
7752 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487753 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327754 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037755
7756 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237757 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037758 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7759 frame.set_rotation(kVideoRotation_270);
7760 video_source_.IncomingCapturedFrame(frame);
7761 WaitForEncodedFrame(timestamp_ms);
7762 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7763
7764 // Set a higher target rate without changing the link_allocation. Should not
7765 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077766 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037768 /*target_bitrate=*/rate,
7769 /*stable_target_bitrate=*/new_stable_rate,
7770 /*link_allocation=*/rate,
7771 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487772 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327773 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037774 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7775 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7776 video_stream_encoder_->Stop();
7777}
7778
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177779TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 14:12:077780 test::ScopedKeyValueConfig field_trials(
7781 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177782 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7783 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7784 const int kFramerateFps = 30;
7785 const int kWidth = 1920;
7786 const int kHeight = 1080;
7787 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7788 // Works on screenshare mode.
7789 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7790 // We rely on the automatic resolution adaptation, but we handle framerate
7791 // adaptation manually by mocking the stats proxy.
7792 video_source_.set_adaptation_enabled(true);
7793
7794 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 16:49:077795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117796 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177797 video_stream_encoder_->SetSource(&video_source_,
7798 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:397799 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177800
7801 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7802 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7803
7804 // Pass enough frames with the full update to trigger animation detection.
7805 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237806 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177807 frame.set_ntp_time_ms(timestamp_ms);
7808 frame.set_timestamp_us(timestamp_ms * 1000);
7809 video_source_.IncomingCapturedFrame(frame);
7810 WaitForEncodedFrame(timestamp_ms);
7811 }
7812
7813 // Resolution should be limited.
7814 rtc::VideoSinkWants expected;
7815 expected.max_framerate_fps = kFramerateFps;
7816 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 14:19:547817 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177818
7819 // Pass one frame with no known update.
7820 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 12:31:237821 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177822 frame.set_ntp_time_ms(timestamp_ms);
7823 frame.set_timestamp_us(timestamp_ms * 1000);
7824 frame.clear_update_rect();
7825
7826 video_source_.IncomingCapturedFrame(frame);
7827 WaitForEncodedFrame(timestamp_ms);
7828
7829 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 12:08:397830 EXPECT_THAT(video_source_.sink_wants(),
7831 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177832
7833 video_stream_encoder_->Stop();
7834}
7835
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327836TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7837 const int kWidth = 720; // 540p adapted down.
7838 const int kHeight = 405;
7839 const int kNumFrames = 3;
7840 // Works on screenshare mode.
7841 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7842 /*num_spatial_layers=*/2, /*screenshare=*/true);
7843
7844 video_source_.set_adaptation_enabled(true);
7845
7846 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117847 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327848
7849 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7850
7851 // Pass enough frames with the full update to trigger animation detection.
7852 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237853 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327854 frame.set_ntp_time_ms(timestamp_ms);
7855 frame.set_timestamp_us(timestamp_ms * 1000);
7856 video_source_.IncomingCapturedFrame(frame);
7857 WaitForEncodedFrame(timestamp_ms);
7858 }
7859
7860 video_stream_encoder_->Stop();
7861}
7862
Yun Zhang1e4d4fd2020-09-30 07:22:467863TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7864 const float downscale_factors[] = {4.0, 2.0, 1.0};
7865 const int number_layers =
7866 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7867 VideoEncoderConfig config;
7868 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7869 for (int i = 0; i < number_layers; ++i) {
7870 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7871 config.simulcast_layers[i].active = true;
7872 }
7873 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:437874 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 07:22:467875 "VP8", /*max qp*/ 56, /*screencast*/ false,
7876 /*screenshare enabled*/ false);
7877 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117878 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7879 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 07:22:467880
7881 // First initialization.
7882 // Encoder should be initialized. Next frame should be key frame.
7883 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7884 sink_.SetNumExpectedLayers(number_layers);
7885 int64_t timestamp_ms = kFrameIntervalMs;
7886 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7887 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117888 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467889 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7890 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7891 VideoFrameType::kVideoFrameKey,
7892 VideoFrameType::kVideoFrameKey}));
7893
7894 // Disable top layer.
7895 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7896 config.simulcast_layers[number_layers - 1].active = false;
7897 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7898 sink_.SetNumExpectedLayers(number_layers - 1);
7899 timestamp_ms += kFrameIntervalMs;
7900 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7901 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117902 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467903 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7904 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7905 VideoFrameType::kVideoFrameDelta,
7906 VideoFrameType::kVideoFrameDelta}));
7907
7908 // Re-enable top layer.
7909 // Encoder should be re-initialized. Next frame should be key frame.
7910 config.simulcast_layers[number_layers - 1].active = true;
7911 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7912 sink_.SetNumExpectedLayers(number_layers);
7913 timestamp_ms += kFrameIntervalMs;
7914 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7915 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117916 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467917 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7918 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7919 VideoFrameType::kVideoFrameKey,
7920 VideoFrameType::kVideoFrameKey}));
7921
7922 // Top layer max rate change.
7923 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7924 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7925 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7926 sink_.SetNumExpectedLayers(number_layers);
7927 timestamp_ms += kFrameIntervalMs;
7928 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7929 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117930 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467931 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7932 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7933 VideoFrameType::kVideoFrameDelta,
7934 VideoFrameType::kVideoFrameDelta}));
7935
7936 // Top layer resolution change.
7937 // Encoder should be re-initialized. Next frame should be key frame.
7938 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7939 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7940 sink_.SetNumExpectedLayers(number_layers);
7941 timestamp_ms += kFrameIntervalMs;
7942 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7943 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117944 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467945 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7946 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7947 VideoFrameType::kVideoFrameKey,
7948 VideoFrameType::kVideoFrameKey}));
7949 video_stream_encoder_->Stop();
7950}
7951
Henrik Boström1124ed12021-02-25 09:30:397952TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7953 const int kFrameWidth = 1280;
7954 const int kFrameHeight = 720;
7955
7956 SetUp();
7957 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117958 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 09:30:397959
7960 // Capturing a frame should reconfigure the encoder and expose the encoder
7961 // resolution, which is the same as the input frame.
7962 int64_t timestamp_ms = kFrameIntervalMs;
7963 video_source_.IncomingCapturedFrame(
7964 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7965 WaitForEncodedFrame(timestamp_ms);
7966 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7967 EXPECT_THAT(video_source_.sink_wants().resolutions,
7968 ::testing::ElementsAreArray(
7969 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7970
7971 video_stream_encoder_->Stop();
7972}
7973
7974TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7975 // Pick downscale factors such that we never encode at full resolution - this
7976 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 14:22:117977 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 09:30:397978 // encoder should not ask for the frame resolution. This allows video frames
7979 // to have the appearence of one resolution but optimize its internal buffers
7980 // for what is actually encoded.
7981 const size_t kNumSimulcastLayers = 3u;
7982 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7983 const int kFrameWidth = 1280;
7984 const int kFrameHeight = 720;
7985 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7986 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7987 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7988 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7989 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7990 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7991
7992 VideoEncoderConfig config;
7993 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7994 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7995 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7996 config.simulcast_layers[i].active = true;
7997 }
7998 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:437999 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 09:30:398000 "VP8", /*max qp*/ 56, /*screencast*/ false,
8001 /*screenshare enabled*/ false);
8002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118003 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8004 0, 0, 0);
Henrik Boström1124ed12021-02-25 09:30:398005
8006 // Capture a frame with all layers active.
8007 int64_t timestamp_ms = kFrameIntervalMs;
8008 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8009 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8010 video_source_.IncomingCapturedFrame(
8011 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8012 WaitForEncodedFrame(timestamp_ms);
8013 // Expect encoded resolutions to match the expected simulcast layers.
8014 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8015 EXPECT_THAT(
8016 video_source_.sink_wants().resolutions,
8017 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8018
8019 // Capture a frame with one of the layers inactive.
8020 timestamp_ms += kFrameIntervalMs;
8021 config.simulcast_layers[2].active = false;
8022 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8023 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8024 video_source_.IncomingCapturedFrame(
8025 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8026 WaitForEncodedFrame(timestamp_ms);
8027
8028 // Expect encoded resolutions to match the expected simulcast layers.
8029 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8030 EXPECT_THAT(video_source_.sink_wants().resolutions,
8031 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8032
8033 // Capture a frame with all but one layer turned off.
8034 timestamp_ms += kFrameIntervalMs;
8035 config.simulcast_layers[1].active = false;
8036 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8037 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8038 video_source_.IncomingCapturedFrame(
8039 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8040 WaitForEncodedFrame(timestamp_ms);
8041
8042 // Expect encoded resolutions to match the expected simulcast layers.
8043 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8044 EXPECT_THAT(video_source_.sink_wants().resolutions,
8045 ::testing::ElementsAreArray({kLayer0Size}));
8046
8047 video_stream_encoder_->Stop();
8048}
8049
Sergey Silkin0e42cf72021-03-15 09:12:578050TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 09:12:578051 ResetEncoder("VP8", 1, 1, 1, false);
8052
Niels Möller8b692902021-06-14 10:04:578053 // Force encoder reconfig.
8054 video_source_.IncomingCapturedFrame(
8055 CreateFrame(1, codec_width_, codec_height_));
8056 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8057
Sergey Silkin0e42cf72021-03-15 09:12:578058 // Set QP on encoded frame and pass the frame to encode complete callback.
8059 // Since QP is present QP parsing won't be triggered and the original value
8060 // should be kept.
8061 EncodedImage encoded_image;
8062 encoded_image.qp_ = 123;
8063 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8064 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8065 CodecSpecificInfo codec_info;
8066 codec_info.codecType = kVideoCodecVP8;
8067 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8068 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8069 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8070 video_stream_encoder_->Stop();
8071}
8072
8073TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 09:12:578074 ResetEncoder("VP8", 1, 1, 1, false);
8075
Niels Möller8b692902021-06-14 10:04:578076 // Force encoder reconfig.
8077 video_source_.IncomingCapturedFrame(
8078 CreateFrame(1, codec_width_, codec_height_));
8079 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8080
Sergey Silkin0e42cf72021-03-15 09:12:578081 // Pass an encoded frame without QP to encode complete callback. QP should be
8082 // parsed and set.
8083 EncodedImage encoded_image;
8084 encoded_image.qp_ = -1;
8085 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8086 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8087 CodecSpecificInfo codec_info;
8088 codec_info.codecType = kVideoCodecVP8;
8089 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8090 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8091 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8092 video_stream_encoder_->Stop();
8093}
8094
8095TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 14:12:078096 webrtc::test::ScopedKeyValueConfig field_trials(
8097 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 09:12:578098
Sergey Silkin0e42cf72021-03-15 09:12:578099 ResetEncoder("VP8", 1, 1, 1, false);
8100
Niels Möller8b692902021-06-14 10:04:578101 // Force encoder reconfig.
8102 video_source_.IncomingCapturedFrame(
8103 CreateFrame(1, codec_width_, codec_height_));
8104 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8105
Sergey Silkin0e42cf72021-03-15 09:12:578106 EncodedImage encoded_image;
8107 encoded_image.qp_ = -1;
8108 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8109 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8110 CodecSpecificInfo codec_info;
8111 codec_info.codecType = kVideoCodecVP8;
8112 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8113 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8114 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8115 video_stream_encoder_->Stop();
8116}
8117
Sergey Silkind19e3b92021-03-16 10:05:308118TEST_F(VideoStreamEncoderTest,
8119 QualityScalingNotAllowed_QualityScalingDisabled) {
8120 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8121
8122 // Disable scaling settings in encoder info.
8123 fake_encoder_.SetQualityScaling(false);
8124 // Disable quality scaling in encoder config.
8125 video_encoder_config.is_quality_scaling_allowed = false;
8126 ConfigureEncoder(std::move(video_encoder_config));
8127
8128 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118129 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:308130
8131 test::FrameForwarder source;
8132 video_stream_encoder_->SetSource(
8133 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8134 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8135 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8136
8137 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8138 WaitForEncodedFrame(1);
8139 video_stream_encoder_->TriggerQualityLow();
8140 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8141
8142 video_stream_encoder_->Stop();
8143}
8144
Qiu Jianlinb54cfde2021-07-29 22:48:038145TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8146 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8147
8148 // Disable scaling settings in encoder info.
8149 fake_encoder_.SetQualityScaling(false);
8150 // Set QP trusted in encoder info.
8151 fake_encoder_.SetIsQpTrusted(true);
8152 // Enable quality scaling in encoder config.
8153 video_encoder_config.is_quality_scaling_allowed = false;
8154 ConfigureEncoder(std::move(video_encoder_config));
8155
8156 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118157 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-29 22:48:038158
8159 test::FrameForwarder source;
8160 video_stream_encoder_->SetSource(
8161 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8162 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8163 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8164
8165 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8166 WaitForEncodedFrame(1);
8167 video_stream_encoder_->TriggerQualityLow();
8168 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8169
8170 video_stream_encoder_->Stop();
8171}
8172
Shuhai Pengf2707702021-09-29 09:19:448173TEST_F(VideoStreamEncoderTest,
8174 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8175 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8176
8177 // Disable scaling settings in encoder info.
8178 fake_encoder_.SetQualityScaling(false);
8179 // Set QP trusted in encoder info.
8180 fake_encoder_.SetIsQpTrusted(true);
8181 // Enable quality scaling in encoder config.
8182 video_encoder_config.is_quality_scaling_allowed = false;
8183 ConfigureEncoder(std::move(video_encoder_config));
8184
8185 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118186 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448187
8188 test::FrameForwarder source;
8189 video_stream_encoder_->SetSource(
8190 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8191 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8192 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8193
8194 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8195 WaitForEncodedFrame(1);
8196 video_stream_encoder_->TriggerQualityLow();
8197 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8198
8199 video_stream_encoder_->Stop();
8200}
8201
8202TEST_F(VideoStreamEncoderTest,
8203 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8204 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8205
8206 // Disable scaling settings in encoder info.
8207 fake_encoder_.SetQualityScaling(false);
8208 // Set QP trusted in encoder info.
8209 fake_encoder_.SetIsQpTrusted(false);
8210 // Enable quality scaling in encoder config.
8211 video_encoder_config.is_quality_scaling_allowed = false;
8212 ConfigureEncoder(std::move(video_encoder_config));
8213
8214 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118215 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448216
8217 test::FrameForwarder source;
8218 video_stream_encoder_->SetSource(
8219 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8220 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8221 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8222
8223 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8224 WaitForEncodedFrame(1);
8225 video_stream_encoder_->TriggerQualityLow();
8226 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8227
8228 video_stream_encoder_->Stop();
8229}
8230
8231TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8232 // Set QP trusted in encoder info.
8233 fake_encoder_.SetIsQpTrusted(false);
8234
8235 const int MinEncBitrateKbps = 30;
8236 const int MaxEncBitrateKbps = 100;
8237 const int MinStartBitrateKbp = 50;
8238 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8239 /*frame_size_pixels=*/codec_width_ * codec_height_,
8240 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8241 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8242 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8243
8244 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118245 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448246
8247 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8248
8249 VideoEncoderConfig video_encoder_config;
8250 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8251 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8252 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8253 MinEncBitrateKbps * 1000;
8254 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8255 kMaxPayloadLength);
8256
8257 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8258 WaitForEncodedFrame(1);
8259 EXPECT_EQ(
8260 MaxEncBitrateKbps,
8261 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8262 EXPECT_EQ(
8263 MinEncBitrateKbps,
8264 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8265
8266 video_stream_encoder_->Stop();
8267}
8268
8269TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8270 // Set QP trusted in encoder info.
8271 fake_encoder_.SetIsQpTrusted(false);
8272
8273 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8274 EncoderInfoSettings::
8275 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8276 codec_width_ * codec_height_,
8277 EncoderInfoSettings::
8278 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8279 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8280
8281 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8282 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8283 const int TargetEncBitrate = MaxEncBitrate;
8284 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8285 DataRate::BitsPerSec(TargetEncBitrate),
8286 DataRate::BitsPerSec(TargetEncBitrate),
8287 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8288
8289 VideoEncoderConfig video_encoder_config;
8290 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8291 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8292 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8293 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8294 kMaxPayloadLength);
8295
8296 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8297 WaitForEncodedFrame(1);
8298 EXPECT_EQ(
8299 MaxEncBitrate / 1000,
8300 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8301 EXPECT_EQ(
8302 MinEncBitrate / 1000,
8303 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8304
8305 video_stream_encoder_->Stop();
8306}
8307
Erik Språnge4589cb2022-04-06 14:44:308308TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8309 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8310 /*num_spatial_layers=*/1,
8311 /*screenshare=*/false, /*allocation_callback_type=*/
8312 VideoStreamEncoder::BitrateAllocationCallbackType::
8313 kVideoBitrateAllocationWhenScreenSharing,
8314 /*num_cores=*/3);
8315
8316 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8317 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8318 video_source_.IncomingCapturedFrame(
8319 CreateFrame(1, /*width=*/320, /*height=*/180));
8320 WaitForEncodedFrame(1);
8321 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8322 VideoCodecComplexity::kComplexityNormal);
8323 video_stream_encoder_->Stop();
8324}
8325
8326TEST_F(VideoStreamEncoderTest,
8327 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8328 webrtc::test::ScopedKeyValueConfig field_trials(
8329 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8330
8331 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8332 /*num_spatial_layers=*/1,
8333 /*screenshare=*/false, /*allocation_callback_type=*/
8334 VideoStreamEncoder::BitrateAllocationCallbackType::
8335 kVideoBitrateAllocationWhenScreenSharing,
8336 /*num_cores=*/2);
8337
8338 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8339 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8340 video_source_.IncomingCapturedFrame(
8341 CreateFrame(1, /*width=*/320, /*height=*/180));
8342 WaitForEncodedFrame(1);
8343 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8344 VideoCodecComplexity::kComplexityNormal);
8345 video_stream_encoder_->Stop();
8346}
8347
8348TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8349 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8350 /*num_spatial_layers=*/1,
8351 /*screenshare=*/false, /*allocation_callback_type=*/
8352 VideoStreamEncoder::BitrateAllocationCallbackType::
8353 kVideoBitrateAllocationWhenScreenSharing,
8354 /*num_cores=*/2);
8355
8356 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8357 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8358 video_source_.IncomingCapturedFrame(
8359 CreateFrame(1, /*width=*/320, /*height=*/180));
8360 WaitForEncodedFrame(1);
8361 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8362 VideoCodecComplexity::kComplexityLow);
8363 video_stream_encoder_->Stop();
8364}
8365
Sergey Silkind19e3b92021-03-16 10:05:308366#if !defined(WEBRTC_IOS)
8367// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8368// disabled by default on iOS.
8369TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8370 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8371
8372 // Disable scaling settings in encoder info.
8373 fake_encoder_.SetQualityScaling(false);
8374 // Enable quality scaling in encoder config.
8375 video_encoder_config.is_quality_scaling_allowed = true;
8376 ConfigureEncoder(std::move(video_encoder_config));
8377
8378 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118379 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:308380
8381 test::FrameForwarder source;
8382 video_stream_encoder_->SetSource(
8383 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8384 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8385 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8386
8387 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8388 WaitForEncodedFrame(1);
8389 video_stream_encoder_->TriggerQualityLow();
8390 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8391
8392 video_stream_encoder_->Stop();
8393}
Qiu Jianlinb54cfde2021-07-29 22:48:038394
8395TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8396 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8397
8398 // Disable scaling settings in encoder info.
8399 fake_encoder_.SetQualityScaling(false);
8400 // Set QP trusted in encoder info.
8401 fake_encoder_.SetIsQpTrusted(true);
8402 // Enable quality scaling in encoder config.
8403 video_encoder_config.is_quality_scaling_allowed = true;
8404 ConfigureEncoder(std::move(video_encoder_config));
8405
8406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118407 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-29 22:48:038408
8409 test::FrameForwarder source;
8410 video_stream_encoder_->SetSource(
8411 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8412 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8413 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8414
8415 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8416 WaitForEncodedFrame(1);
8417 video_stream_encoder_->TriggerQualityLow();
8418 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8419
8420 video_stream_encoder_->Stop();
8421}
Shuhai Pengf2707702021-09-29 09:19:448422
8423TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8424 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8425
8426 // Disable scaling settings in encoder info.
8427 fake_encoder_.SetQualityScaling(false);
8428 // Set QP not trusted in encoder info.
8429 fake_encoder_.SetIsQpTrusted(false);
8430 // Enable quality scaling in encoder config.
8431 video_encoder_config.is_quality_scaling_allowed = true;
8432 ConfigureEncoder(std::move(video_encoder_config));
8433
8434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118435 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448436
8437 test::FrameForwarder source;
8438 video_stream_encoder_->SetSource(
8439 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8440 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8441 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8442
8443 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8444 WaitForEncodedFrame(1);
8445 video_stream_encoder_->TriggerQualityLow();
8446 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8447 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8448 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8449
8450 video_stream_encoder_->Stop();
8451}
8452
8453TEST_F(VideoStreamEncoderTest,
8454 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8455 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8456
8457 // Disable scaling settings in encoder info.
8458 fake_encoder_.SetQualityScaling(false);
8459 // Set QP trusted in encoder info.
8460 fake_encoder_.SetIsQpTrusted(true);
8461 // Enable quality scaling in encoder config.
8462 video_encoder_config.is_quality_scaling_allowed = true;
8463 ConfigureEncoder(std::move(video_encoder_config));
8464
8465 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118466 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448467
8468 test::FrameForwarder source;
8469 video_stream_encoder_->SetSource(
8470 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8471 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8472 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8473
8474 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8475 WaitForEncodedFrame(1);
8476 video_stream_encoder_->TriggerQualityLow();
8477 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8478 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8479
8480 video_stream_encoder_->Stop();
8481}
8482
8483TEST_F(VideoStreamEncoderTest,
8484 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8485 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8486
8487 // Disable scaling settings in encoder info.
8488 fake_encoder_.SetQualityScaling(false);
8489 // Set QP trusted in encoder info.
8490 fake_encoder_.SetIsQpTrusted(false);
8491 // Enable quality scaling in encoder config.
8492 video_encoder_config.is_quality_scaling_allowed = true;
8493 ConfigureEncoder(std::move(video_encoder_config));
8494
8495 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118496 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448497
8498 test::FrameForwarder source;
8499 video_stream_encoder_->SetSource(
8500 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8501 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8502 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8503
8504 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8505 WaitForEncodedFrame(1);
8506 video_stream_encoder_->TriggerQualityLow();
8507 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8508
8509 video_stream_encoder_->Stop();
8510}
8511
Erik Språnge4589cb2022-04-06 14:44:308512#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:308513
Henrik Boström56db9ff2021-03-24 08:06:458514// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8515class VideoStreamEncoderWithRealEncoderTest
8516 : public VideoStreamEncoderTest,
8517 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8518 public:
8519 VideoStreamEncoderWithRealEncoderTest()
8520 : VideoStreamEncoderTest(),
8521 codec_type_(std::get<0>(GetParam())),
8522 allow_i420_conversion_(std::get<1>(GetParam())) {}
8523
8524 void SetUp() override {
8525 VideoStreamEncoderTest::SetUp();
8526 std::unique_ptr<VideoEncoder> encoder;
8527 switch (codec_type_) {
8528 case kVideoCodecVP8:
8529 encoder = VP8Encoder::Create();
8530 break;
8531 case kVideoCodecVP9:
8532 encoder = VP9Encoder::Create();
8533 break;
8534 case kVideoCodecAV1:
philipel95701502022-01-18 17:47:528535 encoder = CreateLibaomAv1EncoderIfSupported();
Henrik Boström56db9ff2021-03-24 08:06:458536 break;
8537 case kVideoCodecH264:
8538 encoder =
8539 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8540 break;
8541 case kVideoCodecMultiplex:
8542 mock_encoder_factory_for_multiplex_ =
8543 std::make_unique<MockVideoEncoderFactory>();
8544 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8545 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8546 .WillRepeatedly([] { return VP8Encoder::Create(); });
8547 encoder = std::make_unique<MultiplexEncoderAdapter>(
8548 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8549 false);
8550 break;
8551 default:
Artem Titovd3251962021-11-15 15:57:078552 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 08:06:458553 }
8554 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8555 }
8556
8557 void TearDown() override {
8558 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 14:22:118559 // Ensure `video_stream_encoder_` is destroyed before
8560 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 08:06:458561 video_stream_encoder_.reset();
8562 VideoStreamEncoderTest::TearDown();
8563 }
8564
8565 protected:
8566 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8567 std::unique_ptr<VideoEncoder> encoder) {
8568 // Configure VSE to use the encoder.
8569 encoder_ = std::move(encoder);
8570 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8571 encoder_.get(), &encoder_selector_);
8572 video_send_config_.encoder_settings.encoder_factory =
8573 encoder_proxy_factory_.get();
8574 VideoEncoderConfig video_encoder_config;
8575 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8576 video_encoder_config_ = video_encoder_config.Copy();
8577 ConfigureEncoder(video_encoder_config_.Copy());
8578
8579 // Set bitrate to ensure frame is not dropped.
8580 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118581 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 08:06:458582 }
8583
8584 const VideoCodecType codec_type_;
8585 const bool allow_i420_conversion_;
8586 NiceMock<MockEncoderSelector> encoder_selector_;
8587 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8588 std::unique_ptr<VideoEncoder> encoder_;
8589 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8590};
8591
8592TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8593 auto native_i420_frame = test::CreateMappableNativeFrame(
8594 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8595 video_source_.IncomingCapturedFrame(native_i420_frame);
8596 WaitForEncodedFrame(codec_width_, codec_height_);
8597
8598 auto mappable_native_buffer =
8599 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8600 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8601 mappable_native_buffer->GetMappedFramedBuffers();
8602 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8603 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8604 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8605 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8606}
8607
8608TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8609 auto native_nv12_frame = test::CreateMappableNativeFrame(
8610 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8611 video_source_.IncomingCapturedFrame(native_nv12_frame);
8612 WaitForEncodedFrame(codec_width_, codec_height_);
8613
8614 auto mappable_native_buffer =
8615 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8616 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8617 mappable_native_buffer->GetMappedFramedBuffers();
8618 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8619 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8620 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8621 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8622
8623 if (!allow_i420_conversion_) {
8624 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8625 }
8626}
8627
Erik Språng7444b192021-06-02 12:02:138628TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8629 if (codec_type_ == kVideoCodecMultiplex) {
8630 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8631 return;
8632 }
8633
8634 const size_t kNumSpatialLayers = 3u;
8635 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8636 const int kFrameWidth = 1280;
8637 const int kFrameHeight = 720;
8638 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8639 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8640 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8641 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8642 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8643 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8644
8645 VideoEncoderConfig config;
8646 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8647 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 08:07:118648 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 12:02:138649 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8650 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8651 vp9_settings.numberOfTemporalLayers = 3;
8652 vp9_settings.automaticResizeOn = false;
8653 config.encoder_specific_settings =
8654 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8655 vp9_settings);
8656 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8657 /*fps=*/30.0,
8658 /*first_active_layer=*/0,
8659 /*num_spatial_layers=*/3,
8660 /*num_temporal_layers=*/3,
8661 /*is_screenshare=*/false);
8662 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8663 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 08:07:118664 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 12:02:138665 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8666 /*fps=*/30.0,
8667 /*first_active_layer=*/0,
8668 /*num_spatial_layers=*/3,
8669 /*num_temporal_layers=*/3,
8670 /*is_screenshare=*/false);
8671 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8672 } else {
8673 // Simulcast for VP8/H264.
8674 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8675 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8676 config.simulcast_layers[i].scale_resolution_down_by =
8677 kDownscaleFactors[i];
8678 config.simulcast_layers[i].active = true;
8679 }
8680 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8681 // Turn off frame dropping to prevent flakiness.
8682 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8683 h264_settings.frameDroppingOn = false;
8684 config.encoder_specific_settings = rtc::make_ref_counted<
8685 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8686 }
8687 }
8688
8689 auto set_layer_active = [&](int layer_idx, bool active) {
8690 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8691 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8692 config.spatial_layers[layer_idx].active = active;
8693 } else {
8694 config.simulcast_layers[layer_idx].active = active;
8695 }
8696 };
8697
8698 config.video_stream_factory =
8699 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8700 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8701 /*screencast*/ false,
8702 /*screenshare enabled*/ false);
8703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118704 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8705 0, 0, 0);
Erik Språng7444b192021-06-02 12:02:138706
8707 // Capture a frame with all layers active.
8708 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8709 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8710 int64_t timestamp_ms = kFrameIntervalMs;
8711 video_source_.IncomingCapturedFrame(
8712 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8713
8714 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8715 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8716
8717 // Capture a frame with one of the layers inactive.
8718 set_layer_active(2, false);
8719 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8720 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8721 timestamp_ms += kFrameIntervalMs;
8722 video_source_.IncomingCapturedFrame(
8723 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8724 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8725
8726 // New target bitrates signaled based on lower resolution.
8727 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8729 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8730 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8731
8732 // Re-enable the top layer.
8733 set_layer_active(2, true);
8734 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8735 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8736 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8737
8738 // Bitrate target adjusted back up to enable HD layer...
8739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8740 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8741 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8742 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8743
8744 // ...then add a new frame.
8745 timestamp_ms += kFrameIntervalMs;
8746 video_source_.IncomingCapturedFrame(
8747 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8748 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8749 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8750
8751 video_stream_encoder_->Stop();
8752}
8753
Henrik Boström56db9ff2021-03-24 08:06:458754std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8755 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8756 VideoCodecType codec_type = std::get<0>(info.param);
8757 bool allow_i420_conversion = std::get<1>(info.param);
8758 std::string str;
8759 switch (codec_type) {
8760 case kVideoCodecGeneric:
8761 str = "Generic";
8762 break;
8763 case kVideoCodecVP8:
8764 str = "VP8";
8765 break;
8766 case kVideoCodecVP9:
8767 str = "VP9";
8768 break;
8769 case kVideoCodecAV1:
8770 str = "AV1";
8771 break;
8772 case kVideoCodecH264:
8773 str = "H264";
8774 break;
8775 case kVideoCodecMultiplex:
8776 str = "Multiplex";
8777 break;
8778 default:
Artem Titovd3251962021-11-15 15:57:078779 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 08:06:458780 }
8781 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8782 return str;
8783}
8784
8785constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8786 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8787constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8788 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8789constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 15:01:078790 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 08:06:458791constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8792 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8793#if defined(WEBRTC_USE_H264)
8794constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8795 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8796
8797// The windows compiler does not tolerate #if statements inside the
8798// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8799// and without H264).
8800INSTANTIATE_TEST_SUITE_P(
8801 All,
8802 VideoStreamEncoderWithRealEncoderTest,
8803 ::testing::Values(kVP8DisallowConversion,
8804 kVP9DisallowConversion,
8805 kAV1AllowConversion,
8806 kMultiplexDisallowConversion,
8807 kH264AllowConversion),
8808 TestParametersVideoCodecAndAllowI420ConversionToString);
8809#else
8810INSTANTIATE_TEST_SUITE_P(
8811 All,
8812 VideoStreamEncoderWithRealEncoderTest,
8813 ::testing::Values(kVP8DisallowConversion,
8814 kVP9DisallowConversion,
8815 kAV1AllowConversion,
8816 kMultiplexDisallowConversion),
8817 TestParametersVideoCodecAndAllowI420ConversionToString);
8818#endif
8819
Åsa Persson4d4f62f2021-09-12 14:14:488820class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8821 protected:
8822 void RunTest(const std::vector<VideoStream>& configs,
8823 const int expected_num_init_encode) {
8824 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 08:07:118825 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 14:14:488826 InsertFrameAndWaitForEncoded();
8827 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8828 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 08:07:118829 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8830 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 14:14:488831
8832 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8833 ConfigureEncoder(configs[1]);
8834 InsertFrameAndWaitForEncoded();
8835 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8836 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 08:07:118837 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 14:14:488838 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 08:07:118839 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 14:14:488840
8841 video_stream_encoder_->Stop();
8842 }
8843
8844 void ConfigureEncoder(const VideoStream& stream) {
8845 VideoEncoderConfig config;
8846 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8847 config.max_bitrate_bps = stream.max_bitrate_bps;
8848 config.simulcast_layers[0] = stream;
8849 config.video_stream_factory =
8850 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8851 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8852 /*conference_mode=*/false);
8853 video_stream_encoder_->ConfigureEncoder(std::move(config),
8854 kMaxPayloadLength);
8855 }
8856
8857 void OnBitrateUpdated(DataRate bitrate) {
8858 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8859 bitrate, bitrate, bitrate, 0, 0, 0);
8860 }
8861
8862 void InsertFrameAndWaitForEncoded() {
8863 timestamp_ms_ += kFrameIntervalMs;
8864 video_source_.IncomingCapturedFrame(
8865 CreateFrame(timestamp_ms_, kWidth, kHeight));
8866 sink_.WaitForEncodedFrame(timestamp_ms_);
8867 }
8868
8869 void ExpectEqual(const VideoCodec& actual,
8870 const VideoStream& expected) const {
8871 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8872 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8873 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8874 static_cast<unsigned int>(expected.min_bitrate_bps));
8875 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8876 static_cast<unsigned int>(expected.max_bitrate_bps));
8877 EXPECT_EQ(actual.simulcastStream[0].width,
8878 kWidth / expected.scale_resolution_down_by);
8879 EXPECT_EQ(actual.simulcastStream[0].height,
8880 kHeight / expected.scale_resolution_down_by);
8881 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8882 expected.num_temporal_layers);
8883 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8884 }
8885
8886 VideoStream DefaultConfig() const {
8887 VideoStream stream;
8888 stream.max_framerate = 25;
8889 stream.min_bitrate_bps = 35000;
8890 stream.max_bitrate_bps = 900000;
8891 stream.scale_resolution_down_by = 1.0;
8892 stream.num_temporal_layers = 1;
8893 stream.bitrate_priority = 1.0;
8894 stream.scalability_mode = "";
8895 return stream;
8896 }
8897
8898 const int kWidth = 640;
8899 const int kHeight = 360;
8900 int64_t timestamp_ms_ = 0;
8901};
8902
8903TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8904 VideoStream config1 = DefaultConfig();
8905 VideoStream config2 = config1;
8906 config2.max_framerate++;
8907
8908 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8909}
8910
8911TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8912 VideoStream config1 = DefaultConfig();
8913 VideoStream config2 = config1;
8914 config2.min_bitrate_bps += 10000;
8915
8916 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8917}
8918
8919TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8920 VideoStream config1 = DefaultConfig();
8921 VideoStream config2 = config1;
8922 config2.max_bitrate_bps += 100000;
8923
8924 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8925}
8926
8927TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8928 VideoStream config1 = DefaultConfig();
8929 VideoStream config2 = config1;
8930 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8931
8932 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8933}
8934
8935TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8936 VideoStream config1 = DefaultConfig();
8937 VideoStream config2 = config1;
8938 config2.scale_resolution_down_by *= 2;
8939
8940 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8941}
8942
8943TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8944 VideoStream config1 = DefaultConfig();
8945 VideoStream config2 = config1;
8946 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8947
8948 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8949}
8950
8951TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8952 VideoStream config1 = DefaultConfig();
8953 VideoStream config2 = config1;
8954 config2.scalability_mode = "L1T2";
8955
8956 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8957}
8958
Tommi62b01db2022-01-25 22:41:228959// Simple test that just creates and then immediately destroys an encoder.
8960// The purpose of the test is to make sure that nothing bad happens if the
8961// initialization step on the encoder queue, doesn't run.
8962TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
8963 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
8964 public:
8965 SuperLazyTaskQueue() = default;
8966 ~SuperLazyTaskQueue() override = default;
8967
8968 private:
8969 void Delete() override { delete this; }
8970 void PostTask(std::unique_ptr<QueuedTask> task) override {
8971 // meh.
8972 }
8973 void PostDelayedTask(std::unique_ptr<QueuedTask> task,
8974 uint32_t milliseconds) override {
8975 ASSERT_TRUE(false);
8976 }
8977 };
8978
8979 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 11:52:488980 test::ScopedKeyValueConfig field_trials;
Tommi62b01db2022-01-25 22:41:228981 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
8982 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
8983 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 11:52:488984 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 22:41:228985 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
8986 time_controller.GetClock());
8987 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
8988 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
8989 CreateBuiltinVideoBitrateAllocatorFactory();
8990 VideoStreamEncoderSettings encoder_settings{
8991 VideoEncoder::Capabilities(/*loss_notification=*/false)};
8992 encoder_settings.encoder_factory = &encoder_factory;
8993 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
8994
8995 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8996 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
8997
8998 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
8999 encoder_queue(new SuperLazyTaskQueue());
9000
9001 // Construct a VideoStreamEncoder instance and let it go out of scope without
9002 // doing anything else (including calling Stop()). This should be fine since
9003 // the posted init task will simply be deleted.
9004 auto encoder = std::make_unique<VideoStreamEncoder>(
9005 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
9006 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get()),
9007 std::move(adapter), std::move(encoder_queue),
9008 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 14:12:079009 kVideoBitrateAllocation,
9010 field_trials);
Tommi62b01db2022-01-25 22:41:229011}
9012
Markus Handellb4e96d42021-11-05 11:00:559013TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9014 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9015 auto* adapter_ptr = adapter.get();
9016 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 10:37:169017 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9018 nullptr;
9019 EXPECT_CALL(*adapter_ptr, Initialize)
9020 .WillOnce(Invoke([&video_stream_encoder_callback](
9021 FrameCadenceAdapterInterface::Callback* callback) {
9022 video_stream_encoder_callback = callback;
9023 }));
9024 TaskQueueBase* encoder_queue = nullptr;
9025 auto video_stream_encoder =
9026 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 11:00:559027
Markus Handelle59fee82021-12-23 08:29:239028 // First a call before we know the frame size and hence cannot compute the
9029 // number of simulcast layers.
9030 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9031 &FrameCadenceAdapterInterface::
9032 ZeroHertzModeParams::num_simulcast_layers,
9033 Eq(0)))));
Markus Handellb4e96d42021-11-05 11:00:559034 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 10:37:169035 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 11:00:559036 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9037 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 08:29:239038 factory.DepleteTaskQueues();
9039
9040 // Then a call as we've computed the number of simulcast layers after a passed
9041 // frame.
9042 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9043 &FrameCadenceAdapterInterface::
9044 ZeroHertzModeParams::num_simulcast_layers,
9045 Gt(0)))));
Markus Handell8d87c462021-12-16 10:37:169046 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 15:07:019047 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 11:00:559048 Mock::VerifyAndClearExpectations(adapter_ptr);
9049
Markus Handelle59fee82021-12-23 08:29:239050 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 10:37:169051 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 11:00:559052 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 10:37:169053 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 11:00:559054 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9055 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 10:37:169056 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 15:07:019057 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 11:00:559058}
9059
9060TEST(VideoStreamEncoderFrameCadenceTest,
9061 ForwardsFramesIntoFrameCadenceAdapter) {
9062 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9063 auto* adapter_ptr = adapter.get();
9064 test::FrameForwarder video_source;
9065 SimpleVideoStreamEncoderFactory factory;
9066 auto video_stream_encoder = factory.Create(std::move(adapter));
9067 video_stream_encoder->SetSource(
9068 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9069
9070 EXPECT_CALL(*adapter_ptr, OnFrame);
9071 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9072 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 10:37:169073 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 11:00:559074}
9075
Markus Handellee225432021-11-29 11:35:129076TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9077 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9078 auto* adapter_ptr = adapter.get();
9079 test::FrameForwarder video_source;
9080 SimpleVideoStreamEncoderFactory factory;
9081 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9082 nullptr;
9083 EXPECT_CALL(*adapter_ptr, Initialize)
9084 .WillOnce(Invoke([&video_stream_encoder_callback](
9085 FrameCadenceAdapterInterface::Callback* callback) {
9086 video_stream_encoder_callback = callback;
9087 }));
9088 TaskQueueBase* encoder_queue = nullptr;
9089 auto video_stream_encoder =
9090 factory.Create(std::move(adapter), &encoder_queue);
9091
9092 // This is just to make the VSE operational. We'll feed a frame directly by
9093 // the callback interface.
9094 video_stream_encoder->SetSource(
9095 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9096
9097 VideoEncoderConfig video_encoder_config;
9098 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9099 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9100 /*max_data_payload_length=*/1000);
9101
9102 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9103 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 10:37:169104 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 11:35:129105 factory.DepleteTaskQueues();
9106}
9107
Markus Handell8d87c462021-12-16 10:37:169108TEST(VideoStreamEncoderFrameCadenceTest,
9109 DeactivatesActivatesLayersOnBitrateChanges) {
9110 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9111 auto* adapter_ptr = adapter.get();
9112 SimpleVideoStreamEncoderFactory factory;
9113 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9114 nullptr;
9115 EXPECT_CALL(*adapter_ptr, Initialize)
9116 .WillOnce(Invoke([&video_stream_encoder_callback](
9117 FrameCadenceAdapterInterface::Callback* callback) {
9118 video_stream_encoder_callback = callback;
9119 }));
9120 TaskQueueBase* encoder_queue = nullptr;
9121 auto video_stream_encoder =
9122 factory.Create(std::move(adapter), &encoder_queue);
9123
9124 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9125 // {150000, 450000}.
9126 VideoEncoderConfig video_encoder_config;
9127 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9128 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9129 kMaxPayloadLength);
9130 // Ensure an encoder is created.
9131 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9132
9133 // Both layers enabled at 1 MBit/s.
9134 video_stream_encoder->OnBitrateUpdated(
9135 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9136 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9137 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9138 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9139 factory.DepleteTaskQueues();
9140 Mock::VerifyAndClearExpectations(adapter_ptr);
9141
9142 // Layer 1 disabled at 200 KBit/s.
9143 video_stream_encoder->OnBitrateUpdated(
9144 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9145 DataRate::KilobitsPerSec(200), 0, 0, 0);
9146 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9147 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9148 factory.DepleteTaskQueues();
9149 Mock::VerifyAndClearExpectations(adapter_ptr);
9150
9151 // All layers off at suspended video.
9152 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9153 DataRate::Zero(), 0, 0, 0);
9154 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9155 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9156 factory.DepleteTaskQueues();
9157 Mock::VerifyAndClearExpectations(adapter_ptr);
9158
9159 // Both layers enabled again back at 1 MBit/s.
9160 video_stream_encoder->OnBitrateUpdated(
9161 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9162 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9163 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9164 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9165 factory.DepleteTaskQueues();
9166}
9167
9168TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9169 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9170 auto* adapter_ptr = adapter.get();
9171 SimpleVideoStreamEncoderFactory factory;
9172 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9173 nullptr;
9174 EXPECT_CALL(*adapter_ptr, Initialize)
9175 .WillOnce(Invoke([&video_stream_encoder_callback](
9176 FrameCadenceAdapterInterface::Callback* callback) {
9177 video_stream_encoder_callback = callback;
9178 }));
9179 TaskQueueBase* encoder_queue = nullptr;
9180 auto video_stream_encoder =
9181 factory.Create(std::move(adapter), &encoder_queue);
9182
9183 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9184 VideoEncoderConfig video_encoder_config;
9185 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9186 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9187 kMaxPayloadLength);
9188 video_stream_encoder->OnBitrateUpdated(
9189 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9190 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9191
9192 // Pass a frame which has unconverged results.
9193 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9194 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9195 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9196 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9197 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9198 CodecSpecificInfo codec_specific;
9199 codec_specific.codecType = kVideoCodecGeneric;
9200 return codec_specific;
9201 }));
9202 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9203 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9204 factory.DepleteTaskQueues();
9205 Mock::VerifyAndClearExpectations(adapter_ptr);
9206 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9207
9208 // Pass a frame which converges in layer 0 and not in layer 1.
9209 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9210 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9211 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9212 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9213 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9214 CodecSpecificInfo codec_specific;
9215 codec_specific.codecType = kVideoCodecGeneric;
9216 return codec_specific;
9217 }));
9218 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9219 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9220 factory.DepleteTaskQueues();
9221 Mock::VerifyAndClearExpectations(adapter_ptr);
9222 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9223}
9224
Markus Handell2e0f4f02021-12-21 18:14:589225TEST(VideoStreamEncoderFrameCadenceTest,
9226 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9227 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9228 auto* adapter_ptr = adapter.get();
9229 MockVideoSourceInterface mock_source;
9230 SimpleVideoStreamEncoderFactory factory;
9231 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9232 nullptr;
9233 EXPECT_CALL(*adapter_ptr, Initialize)
9234 .WillOnce(Invoke([&video_stream_encoder_callback](
9235 FrameCadenceAdapterInterface::Callback* callback) {
9236 video_stream_encoder_callback = callback;
9237 }));
9238 TaskQueueBase* encoder_queue = nullptr;
9239 auto video_stream_encoder =
9240 factory.Create(std::move(adapter), &encoder_queue);
9241 video_stream_encoder->SetSource(
9242 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9243 VideoEncoderConfig config;
9244 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9245 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9246 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9247 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9248 // Ensure the encoder is set up.
9249 factory.DepleteTaskQueues();
9250
Markus Handell818e7fb2021-12-30 12:01:339251 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9252 .WillOnce(Invoke([video_stream_encoder_callback] {
9253 video_stream_encoder_callback->RequestRefreshFrame();
9254 }));
Markus Handell2e0f4f02021-12-21 18:14:589255 EXPECT_CALL(mock_source, RequestRefreshFrame);
9256 video_stream_encoder->SendKeyFrame();
9257 factory.DepleteTaskQueues();
9258 Mock::VerifyAndClearExpectations(adapter_ptr);
9259 Mock::VerifyAndClearExpectations(&mock_source);
9260
Markus Handell818e7fb2021-12-30 12:01:339261 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 18:14:589262 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9263 video_stream_encoder->SendKeyFrame();
9264 factory.DepleteTaskQueues();
9265}
9266
Markus Handell818e7fb2021-12-30 12:01:339267TEST(VideoStreamEncoderFrameCadenceTest,
9268 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9269 SimpleVideoStreamEncoderFactory factory;
9270 auto encoder_queue =
9271 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9272 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9273
9274 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 14:12:079275 test::ScopedKeyValueConfig field_trials(
9276 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 12:01:339277 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 11:52:489278 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9279 field_trials);
Markus Handell818e7fb2021-12-30 12:01:339280 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9281
9282 MockVideoSourceInterface mock_source;
9283 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 14:12:079284 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 12:01:339285
9286 video_stream_encoder->SetSource(
9287 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9288 VideoEncoderConfig config;
9289 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9290 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9291 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9292
9293 // Eventually expect a refresh frame request when requesting a key frame
9294 // before initializing zero-hertz mode. This can happen in reality because the
9295 // threads invoking key frame requests and constraints setup aren't
9296 // synchronized.
9297 EXPECT_CALL(mock_source, RequestRefreshFrame);
9298 video_stream_encoder->SendKeyFrame();
9299 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, 30});
9300 factory.DepleteTaskQueues();
9301}
9302
perkj26091b12016-09-01 08:17:409303} // namespace webrtc