blob: c1d89d73b29425b16a69c646f5a02b58cd9e2d47 [file] [log] [blame]
perkj26091b12016-09-01 08:17:401/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 08:30:3111#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 15:24:5913#include <algorithm>
perkj803d97f2016-11-01 18:45:4614#include <limits>
Danil Chapovalovd3ba2362019-04-10 15:01:2315#include <memory>
Henrik Boström56db9ff2021-03-24 08:06:4516#include <tuple>
Per512ecb32016-09-23 13:52:0617#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 14:30:1619#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 15:07:0120#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 15:01:2321#include "api/task_queue/default_task_queue_factory.h"
Markus Handellb4e96d42021-11-05 11:00:5522#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 09:20:0923#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 10:30:0024#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 08:06:4525#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 10:37:1626#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 18:02:5627#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3128#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 07:15:1329#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 09:36:5530#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 16:10:5731#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 08:06:4532#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 12:57:5733#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 08:30:3134#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 10:56:2035#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 11:02:3636#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 15:26:0137#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 14:30:1638#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 13:59:1239#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 17:11:0040#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 15:50:0041#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 08:06:4542#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
43#include "modules/video_coding/codecs/h264/include/h264.h"
44#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
45#include "modules/video_coding/codecs/vp8/include/vp8.h"
46#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 17:32:3747#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 12:02:1348#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 10:24:3349#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 11:06:5950#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2351#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 09:39:5152#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 14:29:0953#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3154#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 17:11:0055#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 11:13:3256#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 15:20:1757#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 06:51:1058#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3159#include "test/encoder_settings.h"
60#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 08:51:4061#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 14:59:0062#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3163#include "test/gmock.h"
64#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 08:06:4565#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2366#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 07:07:2467#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 11:00:5568#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3169#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 08:17:4070
71namespace webrtc {
72
sprang57c2fff2017-01-16 14:24:0273using ::testing::_;
philipeld9cc8c02019-09-16 12:53:4074using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 12:08:3975using ::testing::Eq;
philipeld9cc8c02019-09-16 12:53:4076using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 12:08:3977using ::testing::Ge;
78using ::testing::Gt;
Markus Handellee225432021-11-29 11:35:1279using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 12:08:3980using ::testing::Le;
81using ::testing::Lt;
philipel9b058032020-02-10 10:30:0082using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 11:00:5583using ::testing::Mock;
philipel9b058032020-02-10 10:30:0084using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 11:00:5585using ::testing::Optional;
philipel9b058032020-02-10 10:30:0086using ::testing::Return;
Per Kjellander4190ce92020-12-15 16:24:5587using ::testing::SizeIs;
philipeld9cc8c02019-09-16 12:53:4088using ::testing::StrictMock;
kthelgason876222f2016-11-29 09:44:1189
perkj803d97f2016-11-01 18:45:4690namespace {
Åsa Persson8c1bf952018-09-13 08:42:1991const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 14:56:0092const int kQpLow = 1;
93const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 08:42:1994const int kMinFramerateFps = 2;
95const int kMinBalancedFramerateFps = 7;
96const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 12:37:0097const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 08:07:1198const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
99const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
100const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
101const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 15:02:22102const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 11:21:07103const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 08:42:19104const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 08:48:48105const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 11:12:19106const VideoEncoder::ResolutionBitrateLimits
107 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 12:37:00110
Asa Persson606d3cb2021-10-04 08:07:11111uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 14:30:16112 0x00, 0x00, 0x03, 0x03, 0xF4,
113 0x05, 0x03, 0xC7, 0xE0, 0x1B,
114 0x41, 0x10, 0x8D, 0x00};
115
Sergey Silkin0e42cf72021-03-15 09:12:57116const uint8_t kCodedFrameVp8Qp25[] = {
117 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
118 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
119 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
120
Markus Handell8d87c462021-12-16 10:37:16121void PassAFrame(
122 TaskQueueBase* encoder_queue,
123 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
124 int64_t ntp_time_ms) {
125 encoder_queue->PostTask(
126 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
127 video_stream_encoder_callback->OnFrame(
128 Timestamp::Millis(ntp_time_ms), 1,
129 VideoFrame::Builder()
130 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
131 /*width=*/16, /*height=*/16))
132 .build());
133 }));
134}
135
perkj803d97f2016-11-01 18:45:46136class TestBuffer : public webrtc::I420Buffer {
137 public:
138 TestBuffer(rtc::Event* event, int width, int height)
139 : I420Buffer(width, height), event_(event) {}
140
141 private:
142 friend class rtc::RefCountedObject<TestBuffer>;
143 ~TestBuffer() override {
144 if (event_)
145 event_->Set();
146 }
147 rtc::Event* const event_;
148};
149
Henrik Boström56db9ff2021-03-24 08:06:45150// A fake native buffer that can't be converted to I420. Upon scaling, it
151// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 13:59:12152class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
153 public:
154 FakeNativeBuffer(rtc::Event* event, int width, int height)
155 : event_(event), width_(width), height_(height) {}
156 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
157 int width() const override { return width_; }
158 int height() const override { return height_; }
159 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
160 return nullptr;
161 }
Henrik Boström56db9ff2021-03-24 08:06:45162 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
163 int offset_x,
164 int offset_y,
165 int crop_width,
166 int crop_height,
167 int scaled_width,
168 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 17:21:43169 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
170 scaled_height);
Henrik Boström56db9ff2021-03-24 08:06:45171 }
Noah Richards51db4212019-06-12 13:59:12172
173 private:
174 friend class rtc::RefCountedObject<FakeNativeBuffer>;
175 ~FakeNativeBuffer() override {
176 if (event_)
177 event_->Set();
178 }
179 rtc::Event* const event_;
180 const int width_;
181 const int height_;
182};
183
Evan Shrubsole895556e2020-10-05 07:15:13184// A fake native buffer that is backed by an NV12 buffer.
185class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
186 public:
187 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
188 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
189
190 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
191 int width() const override { return nv12_buffer_->width(); }
192 int height() const override { return nv12_buffer_->height(); }
193 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
194 return nv12_buffer_->ToI420();
195 }
Evan Shrubsoleb556b082020-10-08 12:56:45196 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
197 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
198 if (absl::c_find(types, Type::kNV12) != types.end()) {
199 return nv12_buffer_;
200 }
201 return nullptr;
202 }
Evan Shrubsole895556e2020-10-05 07:15:13203 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
204
205 private:
206 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
207 ~FakeNV12NativeBuffer() override {
208 if (event_)
209 event_->Set();
210 }
211 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
212 rtc::Event* const event_;
213};
214
Niels Möller7dc26b72017-12-06 09:27:48215class CpuOveruseDetectorProxy : public OveruseFrameDetector {
216 public:
Niels Möllerd1f7eb62018-03-28 14:40:58217 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
218 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 16:49:07219 last_target_framerate_fps_(-1),
220 framerate_updated_event_(true /* manual_reset */,
221 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 09:27:48222 virtual ~CpuOveruseDetectorProxy() {}
223
224 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 11:13:32225 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48226 last_target_framerate_fps_ = framerate_fps;
227 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 16:49:07228 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 09:27:48229 }
230
231 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 11:13:32232 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48233 return last_target_framerate_fps_;
234 }
235
Niels Möller4db138e2018-04-19 07:04:13236 CpuOveruseOptions GetOptions() { return options_; }
237
Henrik Boström381d1092020-05-12 16:49:07238 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
239
Niels Möller7dc26b72017-12-06 09:27:48240 private:
Markus Handella3765182020-07-08 11:13:32241 Mutex lock_;
Niels Möller7dc26b72017-12-06 09:27:48242 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 16:49:07243 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 09:27:48244};
245
Henrik Boström0f0aa9c2020-06-02 11:02:36246class FakeVideoSourceRestrictionsListener
247 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 16:49:07248 public:
Henrik Boström0f0aa9c2020-06-02 11:02:36249 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 16:49:07250 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 11:02:36251 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 16:49:07252 RTC_DCHECK(was_restrictions_updated_);
253 }
254
255 rtc::Event* restrictions_updated_event() {
256 return &restrictions_updated_event_;
257 }
258
Henrik Boström0f0aa9c2020-06-02 11:02:36259 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 16:49:07260 void OnVideoSourceRestrictionsUpdated(
261 VideoSourceRestrictions restrictions,
262 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 09:47:46263 rtc::scoped_refptr<Resource> reason,
264 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 16:49:07265 was_restrictions_updated_ = true;
266 restrictions_updated_event_.Set();
267 }
268
269 private:
270 bool was_restrictions_updated_;
271 rtc::Event restrictions_updated_event_;
272};
273
Evan Shrubsole5fd40602020-05-25 14:19:54274auto WantsFps(Matcher<int> fps_matcher) {
275 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
276 fps_matcher);
277}
278
279auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
280 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
281 AllOf(max_pixel_matcher, Gt(0)));
282}
283
Evan Shrubsole5cd7eb82020-05-25 12:08:39284auto ResolutionMax() {
285 return AllOf(
Evan Shrubsole5fd40602020-05-25 14:19:54286 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 12:08:39287 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
288 Eq(absl::nullopt)));
289}
290
291auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 14:19:54292 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 12:08:39293}
294
295auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 14:19:54296 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 12:08:39297}
298
299auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54300 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 12:08:39301}
302
303auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54304 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 12:08:39305}
306
307auto FpsMaxResolutionMax() {
308 return AllOf(FpsMax(), ResolutionMax());
309}
310
311auto UnlimitedSinkWants() {
312 return AllOf(FpsUnlimited(), ResolutionMax());
313}
314
315auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
316 Matcher<int> fps_range_matcher;
317
318 if (last_frame_pixels <= 320 * 240) {
319 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 15:50:00320 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 12:08:39321 fps_range_matcher = AllOf(Ge(10), Le(15));
322 } else if (last_frame_pixels <= 640 * 480) {
323 fps_range_matcher = Ge(15);
324 } else {
325 fps_range_matcher = Eq(kDefaultFramerate);
326 }
327 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
328 fps_range_matcher);
329}
330
Evan Shrubsole5fd40602020-05-25 14:19:54331auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
332 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
333 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
334}
335
336auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
337 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
338}
339
340auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
341 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
342}
343
344auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
345 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
346 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
347}
348
349auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
350 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
351 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
352}
353
354auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
355 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
356 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
357}
358
359auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
360 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
361 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
362}
363
mflodmancc3d4422017-08-03 15:27:51364class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 18:45:46365 public:
Markus Handell9a478b52021-11-18 15:07:01366 VideoStreamEncoderUnderTest(
367 TimeController* time_controller,
368 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
369 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
370 encoder_queue,
371 SendStatisticsProxy* stats_proxy,
372 const VideoStreamEncoderSettings& settings,
373 VideoStreamEncoder::BitrateAllocationCallbackType
374 allocation_callback_type)
375 : VideoStreamEncoder(time_controller->GetClock(),
376 1 /* number_of_cores */,
377 stats_proxy,
378 settings,
379 std::unique_ptr<OveruseFrameDetector>(
380 overuse_detector_proxy_ =
381 new CpuOveruseDetectorProxy(stats_proxy)),
382 std::move(cadence_adapter),
383 std::move(encoder_queue),
384 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 12:31:23385 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 15:59:05386 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 11:02:36387 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 09:47:10388 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 14:29:22389 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 09:36:55390 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 14:29:22391 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 11:02:36392 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 15:26:01393 }
perkj803d97f2016-11-01 18:45:46394
Henrik Boström381d1092020-05-12 16:49:07395 void SetSourceAndWaitForRestrictionsUpdated(
396 rtc::VideoSourceInterface<VideoFrame>* source,
397 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 11:02:36398 FakeVideoSourceRestrictionsListener listener;
399 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07400 SetSource(source, degradation_preference);
401 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 11:02:36402 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07403 }
404
405 void SetSourceAndWaitForFramerateUpdated(
406 rtc::VideoSourceInterface<VideoFrame>* source,
407 const DegradationPreference& degradation_preference) {
408 overuse_detector_proxy_->framerate_updated_event()->Reset();
409 SetSource(source, degradation_preference);
410 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
411 }
412
413 void OnBitrateUpdatedAndWaitForManagedResources(
414 DataRate target_bitrate,
415 DataRate stable_target_bitrate,
416 DataRate link_allocation,
417 uint8_t fraction_lost,
418 int64_t round_trip_time_ms,
419 double cwnd_reduce_ratio) {
420 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
421 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
422 // Bitrate is updated on the encoder queue.
423 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 16:49:07424 }
425
kthelgason2fc52542017-03-03 08:24:41426 // This is used as a synchronisation mechanism, to make sure that the
427 // encoder queue is not blocked before we start sending it frames.
428 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 09:11:55429 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 08:24:41430 }
431
Henrik Boström91aa7322020-04-28 10:24:33432 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 13:25:32433 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 10:24:33434 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12435 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36436 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33437 event.Set();
438 });
439 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55440 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 10:24:33441 }
Tomas Gunnarsson612445e2020-09-21 12:31:23442
Henrik Boström91aa7322020-04-28 10:24:33443 void TriggerCpuUnderuse() {
444 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12445 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36446 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33447 event.Set();
448 });
449 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55450 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 13:25:32451 }
kthelgason876222f2016-11-29 09:44:11452
Henrik Boström91aa7322020-04-28 10:24:33453 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 13:25:32454 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 10:24:33455 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12456 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36457 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33458 event.Set();
459 });
460 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55461 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 13:25:32462 }
Åsa Perssonb67c44c2019-09-24 13:25:32463 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 10:24:33464 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12465 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36466 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33467 event.Set();
468 });
469 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55470 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 10:24:33471 }
472
Tomas Gunnarsson612445e2020-09-21 12:31:23473 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 09:27:48474 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 14:29:22475 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
476 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 11:02:36477 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 18:45:46478};
479
Noah Richards51db4212019-06-12 13:59:12480// Simulates simulcast behavior and makes highest stream resolutions divisible
481// by 4.
482class CroppingVideoStreamFactory
483 : public VideoEncoderConfig::VideoStreamFactoryInterface {
484 public:
Åsa Persson17b29b92020-10-17 10:57:58485 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 13:59:12486
487 private:
488 std::vector<VideoStream> CreateEncoderStreams(
489 int width,
490 int height,
491 const VideoEncoderConfig& encoder_config) override {
492 std::vector<VideoStream> streams = test::CreateVideoStreams(
493 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 13:59:12494 return streams;
495 }
Noah Richards51db4212019-06-12 13:59:12496};
497
sprangb1ca0732017-02-01 16:38:12498class AdaptingFrameForwarder : public test::FrameForwarder {
499 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23500 explicit AdaptingFrameForwarder(TimeController* time_controller)
501 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 12:51:49502 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 16:38:12503
504 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 11:13:32505 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12506 adaptation_enabled_ = enabled;
507 }
508
asaperssonfab67072017-04-04 12:51:49509 bool adaption_enabled() const {
Markus Handella3765182020-07-08 11:13:32510 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12511 return adaptation_enabled_;
512 }
513
Henrik Boström1124ed12021-02-25 09:30:39514 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
515 // the resolution or frame rate was different than it is currently. If
516 // something else is modified, such as encoder resolutions, but the resolution
517 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-16 06:40:18518 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 11:13:32519 MutexLock lock(&mutex_);
asapersson09f05612017-05-16 06:40:18520 return last_wants_;
521 }
522
Danil Chapovalovb9b146c2018-06-15 10:28:07523 absl::optional<int> last_sent_width() const { return last_width_; }
524 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-09 01:04:29525
sprangb1ca0732017-02-01 16:38:12526 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 12:31:23527 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 09:11:55528 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:23529
sprangb1ca0732017-02-01 16:38:12530 int cropped_width = 0;
531 int cropped_height = 0;
532 int out_width = 0;
533 int out_height = 0;
sprangc5d62e22017-04-03 06:53:04534 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23535 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
536 << "w=" << video_frame.width()
537 << "h=" << video_frame.height();
sprangc5d62e22017-04-03 06:53:04538 if (adapter_.AdaptFrameResolution(
539 video_frame.width(), video_frame.height(),
540 video_frame.timestamp_us() * 1000, &cropped_width,
541 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 22:49:37542 VideoFrame adapted_frame =
543 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43544 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 22:49:37545 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 17:05:30546 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 22:49:37547 .set_timestamp_ms(99)
548 .set_rotation(kVideoRotation_0)
549 .build();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:17550 if (video_frame.has_update_rect()) {
551 adapted_frame.set_update_rect(
552 video_frame.update_rect().ScaleWithFrame(
553 video_frame.width(), video_frame.height(), 0, 0,
554 video_frame.width(), video_frame.height(), out_width,
555 out_height));
556 }
sprangc5d62e22017-04-03 06:53:04557 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-09 01:04:29558 last_width_.emplace(adapted_frame.width());
559 last_height_.emplace(adapted_frame.height());
560 } else {
Danil Chapovalovb9b146c2018-06-15 10:28:07561 last_width_ = absl::nullopt;
562 last_height_ = absl::nullopt;
sprangc5d62e22017-04-03 06:53:04563 }
sprangb1ca0732017-02-01 16:38:12564 } else {
Harald Alvestrand97597c02021-11-04 12:01:23565 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 16:38:12566 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-09 01:04:29567 last_width_.emplace(video_frame.width());
568 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 16:38:12569 }
570 }
571
Åsa Perssonf5f7e8e2021-06-09 20:55:38572 void OnOutputFormatRequest(int width, int height) {
573 absl::optional<std::pair<int, int>> target_aspect_ratio =
574 std::make_pair(width, height);
575 absl::optional<int> max_pixel_count = width * height;
576 absl::optional<int> max_fps;
577 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
578 max_fps);
579 }
580
sprangb1ca0732017-02-01 16:38:12581 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
582 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 11:13:32583 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 09:30:39584 rtc::VideoSinkWants prev_wants = sink_wants_locked();
585 bool did_adapt =
586 prev_wants.max_pixel_count != wants.max_pixel_count ||
587 prev_wants.target_pixel_count != wants.target_pixel_count ||
588 prev_wants.max_framerate_fps != wants.max_framerate_fps;
589 if (did_adapt) {
590 last_wants_ = prev_wants;
591 }
Rasmus Brandt287e4642019-11-15 15:56:01592 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 06:37:30593 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 16:38:12594 }
Tomas Gunnarsson612445e2020-09-21 12:31:23595
596 TimeController* const time_controller_;
sprangb1ca0732017-02-01 16:38:12597 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 11:13:32598 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
599 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 10:28:07600 absl::optional<int> last_width_;
601 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 16:38:12602};
sprangc5d62e22017-04-03 06:53:04603
Niels Möller213618e2018-07-24 07:29:58604// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-03 06:53:04605class MockableSendStatisticsProxy : public SendStatisticsProxy {
606 public:
607 MockableSendStatisticsProxy(Clock* clock,
608 const VideoSendStream::Config& config,
609 VideoEncoderConfig::ContentType content_type)
610 : SendStatisticsProxy(clock, config, content_type) {}
611
612 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 11:13:32613 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04614 if (mock_stats_)
615 return *mock_stats_;
616 return SendStatisticsProxy::GetStats();
617 }
618
Niels Möller213618e2018-07-24 07:29:58619 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 11:13:32620 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 07:29:58621 if (mock_stats_)
622 return mock_stats_->input_frame_rate;
623 return SendStatisticsProxy::GetInputFrameRate();
624 }
sprangc5d62e22017-04-03 06:53:04625 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 11:13:32626 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04627 mock_stats_.emplace(stats);
628 }
629
630 void ResetMockStats() {
Markus Handella3765182020-07-08 11:13:32631 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04632 mock_stats_.reset();
633 }
634
Tomas Gunnarsson612445e2020-09-21 12:31:23635 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
636 on_frame_dropped_ = std::move(callback);
637 }
638
sprangc5d62e22017-04-03 06:53:04639 private:
Tomas Gunnarsson612445e2020-09-21 12:31:23640 void OnFrameDropped(DropReason reason) override {
641 SendStatisticsProxy::OnFrameDropped(reason);
642 if (on_frame_dropped_)
643 on_frame_dropped_(reason);
644 }
645
Markus Handella3765182020-07-08 11:13:32646 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 10:28:07647 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 12:31:23648 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-03 06:53:04649};
650
Markus Handellb4e96d42021-11-05 11:00:55651class SimpleVideoStreamEncoderFactory {
652 public:
653 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
654 public:
655 using VideoStreamEncoder::VideoStreamEncoder;
656 ~AdaptedVideoStreamEncoder() { Stop(); }
657 };
658
Markus Handell8d87c462021-12-16 10:37:16659 class MockFakeEncoder : public test::FakeEncoder {
660 public:
661 using FakeEncoder::FakeEncoder;
662 MOCK_METHOD(CodecSpecificInfo,
663 EncodeHook,
664 (EncodedImage & encoded_image,
665 rtc::scoped_refptr<EncodedImageBuffer> buffer),
666 (override));
667 };
668
Markus Handellee225432021-11-29 11:35:12669 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 11:00:55670 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 11:35:12671 encoder_settings_.bitrate_allocator_factory =
672 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 11:00:55673 }
674
675 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
Markus Handellee225432021-11-29 11:35:12676 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
677 TaskQueueBase** encoder_queue_ptr = nullptr) {
678 auto encoder_queue =
679 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
680 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
681 if (encoder_queue_ptr)
682 *encoder_queue_ptr = encoder_queue.get();
Markus Handellb4e96d42021-11-05 11:00:55683 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
684 time_controller_.GetClock(),
685 /*number_of_cores=*/1,
686 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
687 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 11:35:12688 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 11:00:55689 VideoStreamEncoder::BitrateAllocationCallbackType::
690 kVideoBitrateAllocation);
691 result->SetSink(&sink_, /*rotation_applied=*/false);
692 return result;
693 }
694
Markus Handell9a478b52021-11-18 15:07:01695 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 10:37:16696 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 15:07:01697
Markus Handellb4e96d42021-11-05 11:00:55698 private:
699 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
700 public:
701 ~NullEncoderSink() override = default;
702 void OnEncoderConfigurationChanged(
703 std::vector<VideoStream> streams,
704 bool is_svc,
705 VideoEncoderConfig::ContentType content_type,
706 int min_transmit_bitrate_bps) override {}
707 void OnBitrateAllocationUpdated(
708 const VideoBitrateAllocation& allocation) override {}
709 void OnVideoLayersAllocationUpdated(
710 VideoLayersAllocation allocation) override {}
711 Result OnEncodedImage(
712 const EncodedImage& encoded_image,
713 const CodecSpecificInfo* codec_specific_info) override {
714 return Result(EncodedImageCallback::Result::OK);
715 }
716 };
717
Markus Handellee225432021-11-29 11:35:12718 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
719 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
720 time_controller_.CreateTaskQueueFactory()};
721 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
722 std::make_unique<MockableSendStatisticsProxy>(
723 time_controller_.GetClock(),
724 VideoSendStream::Config(nullptr),
725 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
726 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
727 CreateBuiltinVideoBitrateAllocatorFactory();
728 VideoStreamEncoderSettings encoder_settings_{
729 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 10:37:16730 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
731 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 11:00:55732 NullEncoderSink sink_;
733};
734
735class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
736 public:
737 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 10:37:16738 MOCK_METHOD(void,
739 SetZeroHertzModeEnabled,
740 (absl::optional<ZeroHertzModeParams>),
741 (override));
Markus Handellb4e96d42021-11-05 11:00:55742 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 11:35:12743 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
744 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 10:37:16745 MOCK_METHOD(void,
746 UpdateLayerQualityConvergence,
747 (int spatial_index, bool converged),
748 (override));
749 MOCK_METHOD(void,
750 UpdateLayerStatus,
751 (int spatial_index, bool enabled),
752 (override));
Markus Handell2e0f4f02021-12-21 18:14:58753 MOCK_METHOD(bool, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 11:00:55754};
755
philipel9b058032020-02-10 10:30:00756class MockEncoderSelector
757 : public VideoEncoderFactory::EncoderSelectorInterface {
758 public:
Danil Chapovalov91fdc602020-05-14 17:17:51759 MOCK_METHOD(void,
760 OnCurrentEncoder,
761 (const SdpVideoFormat& format),
762 (override));
763 MOCK_METHOD(absl::optional<SdpVideoFormat>,
764 OnAvailableBitrate,
765 (const DataRate& rate),
766 (override));
767 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 10:30:00768};
769
Markus Handell2e0f4f02021-12-21 18:14:58770class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
771 public:
772 MOCK_METHOD(void,
773 AddOrUpdateSink,
774 (rtc::VideoSinkInterface<VideoFrame>*,
775 const rtc::VideoSinkWants&),
776 (override));
777 MOCK_METHOD(void,
778 RemoveSink,
779 (rtc::VideoSinkInterface<VideoFrame>*),
780 (override));
781 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
782};
783
perkj803d97f2016-11-01 18:45:46784} // namespace
785
mflodmancc3d4422017-08-03 15:27:51786class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 08:17:40787 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23788 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 08:17:40789
mflodmancc3d4422017-08-03 15:27:51790 VideoStreamEncoderTest()
perkj26091b12016-09-01 08:17:40791 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-03 06:45:26792 codec_width_(320),
793 codec_height_(240),
Åsa Persson8c1bf952018-09-13 08:42:19794 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 12:31:23795 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 07:04:13796 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-03 06:53:04797 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 12:31:23798 time_controller_.GetClock(),
perkj803d97f2016-11-01 18:45:46799 video_send_config_,
800 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 12:31:23801 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 08:17:40802
803 void SetUp() override {
perkj803d97f2016-11-01 18:45:46804 metrics::Reset();
perkj26091b12016-09-01 08:17:40805 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 07:04:13806 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56807 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 12:18:34808 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 13:36:51809 video_send_config_.rtp.payload_name = "FAKE";
810 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 08:17:40811
Per512ecb32016-09-23 13:52:06812 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51813 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 06:57:51814 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
815 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
816 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 15:41:30817 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 14:06:52818
Niels Möllerf1338562018-04-26 07:51:47819 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 12:37:00820 }
821
Per Kjellanderb03b6c82021-01-03 09:26:03822 void ConfigureEncoder(
823 VideoEncoderConfig video_encoder_config,
824 VideoStreamEncoder::BitrateAllocationCallbackType
825 allocation_callback_type =
826 VideoStreamEncoder::BitrateAllocationCallbackType::
827 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 15:27:51828 if (video_stream_encoder_)
829 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 15:07:01830
831 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
832 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
833 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
834 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
835 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
836 encoder_queue_ptr);
837 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
838 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
839 stats_proxy_.get(), video_send_config_.encoder_settings,
840 allocation_callback_type);
Asa Persson606d3cb2021-10-04 08:07:11841 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 15:27:51842 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:41843 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 08:07:11844 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 15:27:51845 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:47846 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:51847 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 12:37:00848 }
849
Per Kjellanderb03b6c82021-01-03 09:26:03850 void ResetEncoder(const std::string& payload_name,
851 size_t num_streams,
852 size_t num_temporal_layers,
853 unsigned char num_spatial_layers,
854 bool screenshare,
855 VideoStreamEncoder::BitrateAllocationCallbackType
856 allocation_callback_type =
857 VideoStreamEncoder::BitrateAllocationCallbackType::
858 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 13:36:51859 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 12:37:00860
861 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:51862 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
863 num_streams, &video_encoder_config);
864 for (auto& layer : video_encoder_config.simulcast_layers) {
865 layer.num_temporal_layers = num_temporal_layers;
866 layer.max_framerate = kDefaultFramerate;
867 }
Erik Språngd7329ca2019-02-21 20:19:53868 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 08:07:11869 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 14:06:52870 video_encoder_config.content_type =
871 screenshare ? VideoEncoderConfig::ContentType::kScreen
872 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 07:28:40873 if (payload_name == "VP9") {
874 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
875 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 13:29:23876 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 07:28:40877 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43878 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
879 vp9_settings);
emircanbbcc3562017-08-18 07:28:40880 }
Per Kjellanderb03b6c82021-01-03 09:26:03881 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 08:17:40882 }
883
sprang57c2fff2017-01-16 14:24:02884 VideoFrame CreateFrame(int64_t ntp_time_ms,
885 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 17:05:30886 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43887 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 17:05:30888 destruction_event, codec_width_, codec_height_))
889 .set_ntp_time_ms(ntp_time_ms)
890 .set_timestamp_ms(99)
891 .set_rotation(kVideoRotation_0)
892 .build();
perkj26091b12016-09-01 08:17:40893 }
894
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26895 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
896 rtc::Event* destruction_event,
897 int offset_x) const {
Åsa Persson90719572021-04-08 17:05:30898 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43899 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 17:05:30900 destruction_event, codec_width_, codec_height_))
901 .set_ntp_time_ms(ntp_time_ms)
902 .set_timestamp_ms(99)
903 .set_rotation(kVideoRotation_0)
904 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
905 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26906 }
907
sprang57c2fff2017-01-16 14:24:02908 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 12:02:13909 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
910 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 17:05:30911 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 12:02:13912 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 17:05:30913 .set_ntp_time_ms(ntp_time_ms)
914 .set_timestamp_ms(ntp_time_ms)
915 .set_rotation(kVideoRotation_0)
916 .build();
perkj803d97f2016-11-01 18:45:46917 }
918
Evan Shrubsole895556e2020-10-05 07:15:13919 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 17:05:30920 return VideoFrame::Builder()
921 .set_video_frame_buffer(NV12Buffer::Create(width, height))
922 .set_ntp_time_ms(ntp_time_ms)
923 .set_timestamp_ms(ntp_time_ms)
924 .set_rotation(kVideoRotation_0)
925 .build();
Evan Shrubsole895556e2020-10-05 07:15:13926 }
927
Noah Richards51db4212019-06-12 13:59:12928 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
929 rtc::Event* destruction_event,
930 int width,
931 int height) const {
Åsa Persson90719572021-04-08 17:05:30932 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43933 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 17:05:30934 destruction_event, width, height))
935 .set_ntp_time_ms(ntp_time_ms)
936 .set_timestamp_ms(99)
937 .set_rotation(kVideoRotation_0)
938 .build();
Noah Richards51db4212019-06-12 13:59:12939 }
940
Evan Shrubsole895556e2020-10-05 07:15:13941 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
942 rtc::Event* destruction_event,
943 int width,
944 int height) const {
Åsa Persson90719572021-04-08 17:05:30945 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43946 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 17:05:30947 destruction_event, width, height))
948 .set_ntp_time_ms(ntp_time_ms)
949 .set_timestamp_ms(99)
950 .set_rotation(kVideoRotation_0)
951 .build();
Evan Shrubsole895556e2020-10-05 07:15:13952 }
953
Noah Richards51db4212019-06-12 13:59:12954 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
955 rtc::Event* destruction_event) const {
956 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
957 codec_height_);
958 }
959
Åsa Perssonc29cb2c2019-03-25 11:06:59960 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 16:49:07961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:11962 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 11:06:59963
964 video_source_.IncomingCapturedFrame(
965 CreateFrame(1, codec_width_, codec_height_));
966 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 13:09:05967 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 11:06:59968 }
969
sprang4847ae62017-06-27 14:06:52970 void WaitForEncodedFrame(int64_t expected_ntp_time) {
971 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 12:31:23972 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52973 }
974
975 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
976 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:23977 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52978 return ok;
979 }
980
981 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
982 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 12:31:23983 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52984 }
985
986 void ExpectDroppedFrame() {
987 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 12:31:23988 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52989 }
990
991 bool WaitForFrame(int64_t timeout_ms) {
992 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:23993 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52994 return ok;
995 }
996
perkj26091b12016-09-01 08:17:40997 class TestEncoder : public test::FakeEncoder {
998 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23999 explicit TestEncoder(TimeController* time_controller)
1000 : FakeEncoder(time_controller->GetClock()),
1001 time_controller_(time_controller) {
1002 RTC_DCHECK(time_controller_);
1003 }
perkj26091b12016-09-01 08:17:401004
Erik Språngaed30702018-11-05 11:57:171005 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 11:13:321006 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 15:44:421007 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 14:52:331008 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 11:42:181009 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 14:56:001010 info.scaling_settings = VideoEncoder::ScalingSettings(
1011 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 11:42:181012 }
1013 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 11:06:591014 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1015 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 14:27:351016 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 11:06:591017 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 14:27:351018 for (int tid = 0; tid < num_layers; ++tid)
1019 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 11:06:591020 }
1021 }
Erik Språngaed30702018-11-05 11:57:171022 }
Sergey Silkin6456e352019-07-08 15:56:401023
1024 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 08:47:111025 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 15:50:001026 info.apply_alignment_to_all_simulcast_layers =
1027 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 12:56:451028 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-29 22:48:031029 if (is_qp_trusted_.has_value()) {
1030 info.is_qp_trusted = is_qp_trusted_;
1031 }
Erik Språngaed30702018-11-05 11:57:171032 return info;
kthelgason876222f2016-11-29 09:44:111033 }
1034
Erik Språngb7cb7b52019-02-26 14:52:331035 int32_t RegisterEncodeCompleteCallback(
1036 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 11:13:321037 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331038 encoded_image_callback_ = callback;
1039 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1040 }
1041
perkjfa10b552016-10-03 06:45:261042 void ContinueEncode() { continue_encode_event_.Set(); }
1043
1044 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1045 uint32_t timestamp) const {
Markus Handella3765182020-07-08 11:13:321046 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-03 06:45:261047 EXPECT_EQ(timestamp_, timestamp);
1048 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1049 }
1050
kthelgason2fc52542017-03-03 08:24:411051 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 11:13:321052 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 08:24:411053 quality_scaling_ = b;
1054 }
kthelgasonad9010c2017-02-14 08:46:511055
Rasmus Brandt5cad55b2019-12-19 08:47:111056 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 11:13:321057 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 08:47:111058 requested_resolution_alignment_ = requested_resolution_alignment;
1059 }
1060
Åsa Perssonc5a74ff2020-09-20 15:50:001061 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1062 MutexLock lock(&local_mutex_);
1063 apply_alignment_to_all_simulcast_layers_ = b;
1064 }
1065
Mirta Dvornicicccc1b572019-01-15 11:42:181066 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 11:13:321067 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 11:42:181068 is_hardware_accelerated_ = is_hardware_accelerated;
1069 }
1070
Åsa Perssonc29cb2c2019-03-25 11:06:591071 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1072 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 11:13:321073 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:591074 temporal_layers_supported_[spatial_idx] = supported;
1075 }
1076
Sergey Silkin6456e352019-07-08 15:56:401077 void SetResolutionBitrateLimits(
1078 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 11:13:321079 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 15:56:401080 resolution_bitrate_limits_ = thresholds;
1081 }
1082
sprangfe627f32017-03-29 15:24:591083 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 11:13:321084 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 15:24:591085 force_init_encode_failed_ = force_failure;
1086 }
1087
Niels Möller6bb5ab92019-01-11 10:11:101088 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 11:13:321089 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:101090 rate_factor_ = rate_factor;
1091 }
1092
Erik Språngd7329ca2019-02-21 20:19:531093 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 11:13:321094 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:101095 return last_framerate_;
1096 }
1097
Erik Språngd7329ca2019-02-21 20:19:531098 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 11:13:321099 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:261100 return last_update_rect_;
1101 }
1102
Niels Möller87e2d782019-03-07 09:18:231103 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 11:13:321104 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531105 return last_frame_types_;
1106 }
1107
1108 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 09:18:231109 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 14:43:581110 keyframe ? VideoFrameType::kVideoFrameKey
1111 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:531112 {
Markus Handella3765182020-07-08 11:13:321113 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531114 last_frame_types_ = frame_type;
1115 }
Niels Möllerb859b322019-03-07 11:40:011116 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 20:19:531117 }
1118
Sergey Silkin0e42cf72021-03-15 09:12:571119 void InjectEncodedImage(const EncodedImage& image,
1120 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 11:13:321121 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 09:12:571122 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 14:52:331123 }
1124
Mirta Dvornicic97910da2020-07-14 13:29:231125 void SetEncodedImageData(
1126 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 11:13:321127 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 13:29:231128 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 14:30:161129 }
1130
Erik Språngd7329ca2019-02-21 20:19:531131 void ExpectNullFrame() {
Markus Handella3765182020-07-08 11:13:321132 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531133 expect_null_frame_ = true;
1134 }
1135
Erik Språng5056af02019-09-02 13:53:111136 absl::optional<VideoEncoder::RateControlParameters>
1137 GetAndResetLastRateControlSettings() {
1138 auto settings = last_rate_control_settings_;
1139 last_rate_control_settings_.reset();
1140 return settings;
Erik Språngd7329ca2019-02-21 20:19:531141 }
1142
Henrik Boström56db9ff2021-03-24 08:06:451143 int GetLastInputWidth() const {
1144 MutexLock lock(&local_mutex_);
1145 return last_input_width_;
1146 }
1147
1148 int GetLastInputHeight() const {
1149 MutexLock lock(&local_mutex_);
1150 return last_input_height_;
1151 }
1152
Evan Shrubsole895556e2020-10-05 07:15:131153 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1154 MutexLock lock(&local_mutex_);
1155 return last_input_pixel_format_;
1156 }
1157
Evan Shrubsole7c079f62019-09-26 07:55:031158 int GetNumSetRates() const {
Markus Handella3765182020-07-08 11:13:321159 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031160 return num_set_rates_;
1161 }
1162
Evan Shrubsoleb556b082020-10-08 12:56:451163 void SetPreferredPixelFormats(
1164 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1165 pixel_formats) {
1166 MutexLock lock(&local_mutex_);
1167 preferred_pixel_formats_ = std::move(pixel_formats);
1168 }
1169
Qiu Jianlinb54cfde2021-07-29 22:48:031170 void SetIsQpTrusted(absl::optional<bool> trusted) {
1171 MutexLock lock(&local_mutex_);
1172 is_qp_trusted_ = trusted;
1173 }
1174
perkjfa10b552016-10-03 06:45:261175 private:
perkj26091b12016-09-01 08:17:401176 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 09:18:231177 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 08:17:401178 {
Markus Handella3765182020-07-08 11:13:321179 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531180 if (expect_null_frame_) {
1181 EXPECT_EQ(input_image.timestamp(), 0u);
1182 EXPECT_EQ(input_image.width(), 1);
1183 last_frame_types_ = *frame_types;
1184 expect_null_frame_ = false;
1185 } else {
1186 EXPECT_GT(input_image.timestamp(), timestamp_);
1187 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1188 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1189 }
perkj26091b12016-09-01 08:17:401190
1191 timestamp_ = input_image.timestamp();
1192 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 18:45:461193 last_input_width_ = input_image.width();
1194 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:261195 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 20:19:531196 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 07:15:131197 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 08:17:401198 }
Niels Möllerb859b322019-03-07 11:40:011199 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 08:17:401200 return result;
1201 }
1202
Niels Möller08ae7ce2020-09-23 13:58:121203 CodecSpecificInfo EncodeHook(
1204 EncodedImage& encoded_image,
1205 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 15:30:361206 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231207 {
1208 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 15:30:361209 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 13:29:231210 }
1211 MutexLock lock(&local_mutex_);
1212 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 15:30:361213 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 13:29:231214 }
Danil Chapovalov2549f172020-08-12 15:30:361215 return codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231216 }
1217
sprangfe627f32017-03-29 15:24:591218 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 12:57:571219 const Settings& settings) override {
1220 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 12:18:341221
Markus Handella3765182020-07-08 11:13:321222 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331223 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 12:18:341224
Erik Språng82fad3d2018-03-21 08:57:231225 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 15:24:591226 // Simulate setting up temporal layers, in order to validate the life
1227 // cycle of these objects.
Elad Aloncde8ab22019-03-20 10:56:201228 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 09:20:091229 frame_buffer_controller_ =
1230 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 15:24:591231 }
Erik Språngb7cb7b52019-02-26 14:52:331232 if (force_init_encode_failed_) {
1233 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 15:24:591234 return -1;
Erik Språngb7cb7b52019-02-26 14:52:331235 }
Mirta Dvornicicccc1b572019-01-15 11:42:181236
Erik Språngb7cb7b52019-02-26 14:52:331237 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 15:24:591238 return res;
1239 }
1240
Erik Språngb7cb7b52019-02-26 14:52:331241 int32_t Release() override {
Markus Handella3765182020-07-08 11:13:321242 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331243 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1244 initialized_ = EncoderState::kUninitialized;
1245 return FakeEncoder::Release();
1246 }
1247
Erik Språng16cb8f52019-04-12 11:59:091248 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 11:13:321249 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031250 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 10:11:101251 VideoBitrateAllocation adjusted_rate_allocation;
1252 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1253 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 11:59:091254 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 10:11:101255 adjusted_rate_allocation.SetBitrate(
1256 si, ti,
Erik Språng16cb8f52019-04-12 11:59:091257 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 10:11:101258 rate_factor_));
1259 }
1260 }
1261 }
Erik Språng16cb8f52019-04-12 11:59:091262 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 13:53:111263 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 11:59:091264 RateControlParameters adjusted_paramters = parameters;
1265 adjusted_paramters.bitrate = adjusted_rate_allocation;
1266 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 10:11:101267 }
1268
Tomas Gunnarsson612445e2020-09-21 12:31:231269 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321270 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 14:52:331271 enum class EncoderState {
1272 kUninitialized,
1273 kInitializationFailed,
1274 kInitialized
Markus Handella3765182020-07-08 11:13:321275 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 08:17:401276 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 11:13:321277 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1278 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1279 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1280 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1281 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1282 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 15:50:001283 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1284 false;
Markus Handella3765182020-07-08 11:13:321285 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 13:29:231286 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1287 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 10:56:201288 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 11:13:321289 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:591290 absl::optional<bool>
1291 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 11:13:321292 local_mutex_);
1293 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1294 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1295 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 13:53:111296 absl::optional<VideoEncoder::RateControlParameters>
1297 last_rate_control_settings_;
Markus Handella3765182020-07-08 11:13:321298 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1299 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 09:18:231300 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 20:19:531301 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 11:13:321302 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1303 nullptr;
Evan Shrubsolefb862742020-03-16 15:18:361304 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 15:56:401305 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 11:13:321306 RTC_GUARDED_BY(local_mutex_);
1307 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 07:15:131308 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1309 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 12:56:451310 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1311 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-29 22:48:031312 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 08:17:401313 };
1314
mflodmancc3d4422017-08-03 15:27:511315 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 08:17:401316 public:
Tomas Gunnarsson612445e2020-09-21 12:31:231317 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1318 : time_controller_(time_controller), test_encoder_(test_encoder) {
1319 RTC_DCHECK(time_controller_);
1320 }
perkj26091b12016-09-01 08:17:401321
perkj26091b12016-09-01 08:17:401322 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 14:06:521323 EXPECT_TRUE(
1324 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1325 }
1326
1327 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1328 int64_t timeout_ms) {
perkj26091b12016-09-01 08:17:401329 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:231330 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 14:06:521331 return false;
perkj26091b12016-09-01 08:17:401332 {
Markus Handella3765182020-07-08 11:13:321333 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121334 timestamp = last_timestamp_;
perkj26091b12016-09-01 08:17:401335 }
1336 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 14:06:521337 return true;
perkj26091b12016-09-01 08:17:401338 }
1339
sprangb1ca0732017-02-01 16:38:121340 void WaitForEncodedFrame(uint32_t expected_width,
1341 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 12:31:231342 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 13:13:561343 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-03 06:53:041344 }
1345
Åsa Perssonc74d8da2017-12-04 13:13:561346 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-03 06:53:041347 uint32_t expected_height) {
sprangb1ca0732017-02-01 16:38:121348 uint32_t width = 0;
1349 uint32_t height = 0;
sprangb1ca0732017-02-01 16:38:121350 {
Markus Handella3765182020-07-08 11:13:321351 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121352 width = last_width_;
1353 height = last_height_;
1354 }
1355 EXPECT_EQ(expected_height, height);
1356 EXPECT_EQ(expected_width, width);
1357 }
1358
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141359 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1360 VideoRotation rotation;
1361 {
Markus Handella3765182020-07-08 11:13:321362 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141363 rotation = last_rotation_;
1364 }
1365 EXPECT_EQ(expected_rotation, rotation);
1366 }
1367
Tomas Gunnarsson612445e2020-09-21 12:31:231368 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 15:02:221369
sprangc5d62e22017-04-03 06:53:041370 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 12:31:231371 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 09:11:551372 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:231373 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 09:11:551374 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:231375 return ret;
sprangc5d62e22017-04-03 06:53:041376 }
1377
perkj26091b12016-09-01 08:17:401378 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 11:13:321379 MutexLock lock(&mutex_);
perkj26091b12016-09-01 08:17:401380 expect_frames_ = false;
1381 }
1382
asaperssonfab67072017-04-04 12:51:491383 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 11:13:321384 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061385 return number_of_reconfigurations_;
1386 }
1387
asaperssonfab67072017-04-04 12:51:491388 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 11:13:321389 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061390 return min_transmit_bitrate_bps_;
1391 }
1392
Erik Språngd7329ca2019-02-21 20:19:531393 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 11:13:321394 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 20:19:531395 num_expected_layers_ = num_layers;
1396 }
1397
Erik Språngb7cb7b52019-02-26 14:52:331398 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 11:13:321399 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331400 return last_capture_time_ms_;
1401 }
1402
Sergey Silkin0e42cf72021-03-15 09:12:571403 const EncodedImage& GetLastEncodedImage() {
1404 MutexLock lock(&mutex_);
1405 return last_encoded_image_;
1406 }
1407
Mirta Dvornicic28f0eb22019-05-28 14:30:161408 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 11:13:321409 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 14:30:161410 return std::move(last_encoded_image_data_);
1411 }
1412
Per Kjellanderdcef6412020-10-07 13:09:051413 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1414 MutexLock lock(&mutex_);
1415 return last_bitrate_allocation_;
1416 }
1417
1418 int number_of_bitrate_allocations() const {
1419 MutexLock lock(&mutex_);
1420 return number_of_bitrate_allocations_;
1421 }
1422
Per Kjellandera9434842020-10-15 15:53:221423 VideoLayersAllocation GetLastVideoLayersAllocation() {
1424 MutexLock lock(&mutex_);
1425 return last_layers_allocation_;
1426 }
1427
1428 int number_of_layers_allocations() const {
1429 MutexLock lock(&mutex_);
1430 return number_of_layers_allocations_;
1431 }
1432
perkj26091b12016-09-01 08:17:401433 private:
sergeyu2cb155a2016-11-04 18:39:291434 Result OnEncodedImage(
1435 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 15:30:361436 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 11:13:321437 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061438 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 09:12:571439 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 14:30:161440 last_encoded_image_data_ = std::vector<uint8_t>(
1441 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 20:19:531442 uint32_t timestamp = encoded_image.Timestamp();
1443 if (last_timestamp_ != timestamp) {
1444 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 12:02:131445 last_width_ = encoded_image._encodedWidth;
1446 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 20:19:531447 } else {
1448 ++num_received_layers_;
Erik Språng7444b192021-06-02 12:02:131449 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1450 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 20:19:531451 }
1452 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 14:52:331453 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141454 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 20:19:531455 if (num_received_layers_ == num_expected_layers_) {
1456 encoded_frame_event_.Set();
1457 }
sprangb1ca0732017-02-01 16:38:121458 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 13:52:061459 }
1460
Rasmus Brandtc402dbe2019-02-04 10:09:461461 void OnEncoderConfigurationChanged(
1462 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 12:10:271463 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 10:09:461464 VideoEncoderConfig::ContentType content_type,
1465 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 11:13:321466 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061467 ++number_of_reconfigurations_;
1468 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1469 }
1470
Per Kjellanderdcef6412020-10-07 13:09:051471 void OnBitrateAllocationUpdated(
1472 const VideoBitrateAllocation& allocation) override {
1473 MutexLock lock(&mutex_);
1474 ++number_of_bitrate_allocations_;
1475 last_bitrate_allocation_ = allocation;
1476 }
1477
Per Kjellandera9434842020-10-15 15:53:221478 void OnVideoLayersAllocationUpdated(
1479 VideoLayersAllocation allocation) override {
1480 MutexLock lock(&mutex_);
1481 ++number_of_layers_allocations_;
1482 last_layers_allocation_ = allocation;
1483 rtc::StringBuilder log;
1484 for (const auto& layer : allocation.active_spatial_layers) {
1485 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1486 << "[";
1487 for (const auto target_bitrate :
1488 layer.target_bitrate_per_temporal_layer) {
1489 log << target_bitrate.kbps() << ",";
1490 }
1491 log << "]";
1492 }
Harald Alvestrand97597c02021-11-04 12:01:231493 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 15:53:221494 }
1495
Tomas Gunnarsson612445e2020-09-21 12:31:231496 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321497 mutable Mutex mutex_;
perkj26091b12016-09-01 08:17:401498 TestEncoder* test_encoder_;
1499 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 09:12:571500 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 14:30:161501 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 16:38:121502 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 14:52:331503 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 16:38:121504 uint32_t last_height_ = 0;
1505 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141506 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 20:19:531507 size_t num_expected_layers_ = 1;
1508 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 08:17:401509 bool expect_frames_ = true;
Per512ecb32016-09-23 13:52:061510 int number_of_reconfigurations_ = 0;
1511 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 13:09:051512 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1513 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 15:53:221514 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1515 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 08:17:401516 };
1517
Sergey Silkin5ee69672019-07-02 12:18:341518 class VideoBitrateAllocatorProxyFactory
1519 : public VideoBitrateAllocatorFactory {
1520 public:
1521 VideoBitrateAllocatorProxyFactory()
1522 : bitrate_allocator_factory_(
1523 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1524
1525 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1526 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 11:13:321527 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341528 codec_config_ = codec;
1529 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1530 }
1531
1532 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 11:13:321533 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341534 return codec_config_;
1535 }
1536
1537 private:
1538 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1539
Markus Handella3765182020-07-08 11:13:321540 mutable Mutex mutex_;
1541 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341542 };
1543
Tomas Gunnarsson612445e2020-09-21 12:31:231544 Clock* clock() { return time_controller_.GetClock(); }
1545 void AdvanceTime(TimeDelta duration) {
1546 time_controller_.AdvanceTime(duration);
1547 }
1548
1549 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1550
1551 protected:
1552 virtual TaskQueueFactory* GetTaskQueueFactory() {
1553 return time_controller_.GetTaskQueueFactory();
1554 }
1555
1556 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 08:17:401557 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 15:41:301558 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 13:52:061559 int codec_width_;
1560 int codec_height_;
sprang4847ae62017-06-27 14:06:521561 int max_framerate_;
perkj26091b12016-09-01 08:17:401562 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 07:07:241563 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 12:18:341564 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-03 06:53:041565 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 08:17:401566 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 12:31:231567 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 15:27:511568 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 08:17:401569};
1570
mflodmancc3d4422017-08-03 15:27:511571TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 16:49:071572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111573 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 07:43:501574 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411575 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 14:06:521576 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 14:53:411577 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 15:27:511578 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401579}
1580
mflodmancc3d4422017-08-03 15:27:511581TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 08:17:401582 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 07:43:501583 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 11:05:491584 // The encoder will cache up to one frame for a short duration. Adding two
1585 // frames means that the first frame will be dropped and the second frame will
1586 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 14:53:411587 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 12:31:231588 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 11:05:491589 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 09:11:551590 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 14:53:411591 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401592
Henrik Boström381d1092020-05-12 16:49:071593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111594 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401595
Sebastian Janssona3177052018-04-10 11:05:491596 // The pending frame should be received.
sprang4847ae62017-06-27 14:06:521597 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 11:05:491598 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1599
1600 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:511601 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401602}
1603
mflodmancc3d4422017-08-03 15:27:511604TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 16:49:071605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111606 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 14:53:411607 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521608 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401609
Henrik Boström381d1092020-05-12 16:49:071610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111611 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1612
Sebastian Janssona3177052018-04-10 11:05:491613 // The encoder will cache up to one frame for a short duration. Adding two
1614 // frames means that the first frame will be dropped and the second frame will
1615 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 14:53:411616 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 11:05:491617 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401618
Henrik Boström381d1092020-05-12 16:49:071619 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111620 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:521621 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 11:05:491622 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1623 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 15:27:511624 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401625}
1626
mflodmancc3d4422017-08-03 15:27:511627TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 16:49:071628 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111629 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 14:53:411630 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521631 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401632
1633 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 14:53:411634 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 08:17:401635
perkja49cbd32016-09-16 14:53:411636 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521637 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511638 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401639}
1640
mflodmancc3d4422017-08-03 15:27:511641TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 16:49:071642 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111643 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401644
perkja49cbd32016-09-16 14:53:411645 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521646 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401647
mflodmancc3d4422017-08-03 15:27:511648 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401649 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 07:43:501650 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411651 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1652 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401653}
1654
Markus Handell9a478b52021-11-18 15:07:011655TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1656 test::FrameForwarder source;
1657 video_stream_encoder_->SetSource(&source,
1658 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 16:49:071659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111660 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401661
Tomas Gunnarsson612445e2020-09-21 12:31:231662 int dropped_count = 0;
1663 stats_proxy_->SetDroppedFrameCallback(
1664 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1665 ++dropped_count;
1666 });
1667
Markus Handell9a478b52021-11-18 15:07:011668 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1669 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1670 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511671 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 12:31:231672 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 08:17:401673}
1674
Henrik Boström56db9ff2021-03-24 08:06:451675TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 16:49:071676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111677 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121678
1679 rtc::Event frame_destroyed_event;
1680 video_source_.IncomingCapturedFrame(
1681 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 08:06:451682 WaitForEncodedFrame(1);
1683 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1684 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 08:07:111685 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1686 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 13:59:121687 video_stream_encoder_->Stop();
1688}
1689
Henrik Boström56db9ff2021-03-24 08:06:451690TEST_F(VideoStreamEncoderTest,
1691 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 13:59:121692 // Use the cropping factory.
1693 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:431694 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 13:59:121695 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1696 kMaxPayloadLength);
1697 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1698
1699 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 16:49:071700 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111701 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121702 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1703 WaitForEncodedFrame(1);
1704 // The encoder will have been configured once.
1705 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111706 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1707 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 13:59:121708
1709 // Now send in a fake frame that needs to be cropped as the width/height
1710 // aren't divisible by 4 (see CreateEncoderStreams above).
1711 rtc::Event frame_destroyed_event;
1712 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1713 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 08:06:451714 WaitForEncodedFrame(2);
1715 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1716 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 08:07:111717 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1718 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 13:59:121719 video_stream_encoder_->Stop();
1720}
1721
Evan Shrubsole895556e2020-10-05 07:15:131722TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111724 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 07:15:131725
1726 video_source_.IncomingCapturedFrame(
1727 CreateNV12Frame(1, codec_width_, codec_height_));
1728 WaitForEncodedFrame(1);
1729 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1730 fake_encoder_.GetLastInputPixelFormat());
1731 video_stream_encoder_->Stop();
1732}
1733
Henrik Boström56db9ff2021-03-24 08:06:451734TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 12:56:451735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111736 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451737
1738 fake_encoder_.SetPreferredPixelFormats({});
1739
1740 rtc::Event frame_destroyed_event;
1741 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1742 1, &frame_destroyed_event, codec_width_, codec_height_));
1743 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451744 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451745 fake_encoder_.GetLastInputPixelFormat());
1746 video_stream_encoder_->Stop();
1747}
1748
Henrik Boström56db9ff2021-03-24 08:06:451749TEST_F(VideoStreamEncoderTest,
1750 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 12:56:451751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111752 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451753
1754 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1755
1756 rtc::Event frame_destroyed_event;
1757 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1758 1, &frame_destroyed_event, codec_width_, codec_height_));
1759 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451760 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451761 fake_encoder_.GetLastInputPixelFormat());
1762 video_stream_encoder_->Stop();
1763}
1764
Henrik Boström56db9ff2021-03-24 08:06:451765TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 12:56:451766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451768
1769 // Fake NV12 native frame does not allow mapping to I444.
1770 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1771
1772 rtc::Event frame_destroyed_event;
1773 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1774 1, &frame_destroyed_event, codec_width_, codec_height_));
1775 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451776 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451777 fake_encoder_.GetLastInputPixelFormat());
1778 video_stream_encoder_->Stop();
1779}
1780
Henrik Boström56db9ff2021-03-24 08:06:451781TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 07:15:131782 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111783 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 07:15:131784
1785 rtc::Event frame_destroyed_event;
1786 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1787 1, &frame_destroyed_event, codec_width_, codec_height_));
1788 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451789 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 07:15:131790 fake_encoder_.GetLastInputPixelFormat());
1791 video_stream_encoder_->Stop();
1792}
1793
Ying Wang9b881ab2020-02-07 13:29:321794TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 16:49:071795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111796 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 13:29:321797 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1798 WaitForEncodedFrame(1);
1799
Henrik Boström381d1092020-05-12 16:49:071800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 13:29:321802 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1803 // frames. Adding two frames means that the first frame will be dropped and
1804 // the second frame will be sent to the encoder.
1805 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1806 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1807 WaitForEncodedFrame(3);
1808 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1809 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1810 WaitForEncodedFrame(5);
1811 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1812 video_stream_encoder_->Stop();
1813}
1814
mflodmancc3d4422017-08-03 15:27:511815TEST_F(VideoStreamEncoderTest,
1816 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 16:49:071817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111818 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 20:37:571819 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061820
1821 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551822 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521823 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571824 // The encoder will have been configured once when the first frame is
1825 // received.
1826 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061827
1828 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:511829 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 13:52:061830 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 15:27:511831 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:471832 kMaxPayloadLength);
Per512ecb32016-09-23 13:52:061833
1834 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551835 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521836 WaitForEncodedFrame(2);
Per21d45d22016-10-30 20:37:571837 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-30 06:25:401838 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-30 05:39:101839
mflodmancc3d4422017-08-03 15:27:511840 video_stream_encoder_->Stop();
perkj26105b42016-09-30 05:39:101841}
1842
mflodmancc3d4422017-08-03 15:27:511843TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 16:49:071844 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111845 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-03 06:45:261846
1847 // Capture a frame and wait for it to synchronize with the encoder thread.
1848 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521849 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571850 // The encoder will have been configured once.
1851 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111852 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1853 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-03 06:45:261854
1855 codec_width_ *= 2;
1856 codec_height_ *= 2;
1857 // Capture a frame with a higher resolution and wait for it to synchronize
1858 // with the encoder thread.
1859 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521860 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 08:07:111861 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1862 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 20:37:571863 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261864
mflodmancc3d4422017-08-03 15:27:511865 video_stream_encoder_->Stop();
perkjfa10b552016-10-03 06:45:261866}
1867
Sergey Silkin443b7ee2019-06-28 10:53:071868TEST_F(VideoStreamEncoderTest,
1869 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 16:49:071870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111871 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 10:53:071872
1873 // Capture a frame and wait for it to synchronize with the encoder thread.
1874 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1875 WaitForEncodedFrame(1);
1876
1877 VideoEncoderConfig video_encoder_config;
1878 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1879 // Changing the max payload data length recreates encoder.
1880 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1881 kMaxPayloadLength / 2);
1882
1883 // Capture a frame and wait for it to synchronize with the encoder thread.
1884 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1885 WaitForEncodedFrame(2);
1886 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1887
1888 video_stream_encoder_->Stop();
1889}
1890
Sergey Silkin5ee69672019-07-02 12:18:341891TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 16:49:071892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111893 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 12:18:341894
1895 VideoEncoderConfig video_encoder_config;
1896 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:111897 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1898 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 12:18:341899 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1900 kMaxPayloadLength);
1901
1902 // Capture a frame and wait for it to synchronize with the encoder thread.
1903 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1904 WaitForEncodedFrame(1);
1905 // The encoder will have been configured once when the first frame is
1906 // received.
1907 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111908 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 12:18:341909 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111910 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 12:18:341911 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1912
Sergey Silkin6456e352019-07-08 15:56:401913 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1914 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 08:07:111915 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1916 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 12:18:341917 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1918 kMaxPayloadLength);
1919
1920 // Capture a frame and wait for it to synchronize with the encoder thread.
1921 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1922 WaitForEncodedFrame(2);
1923 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1924 // Bitrate limits have changed - rate allocator should be reconfigured,
1925 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 08:07:111926 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 12:18:341927 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111928 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 12:18:341929 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111930 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 12:18:341931
1932 video_stream_encoder_->Stop();
1933}
1934
Sergey Silkin6456e352019-07-08 15:56:401935TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 13:48:401936 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 16:49:071937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111938 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401939
Sergey Silkincd02eba2020-01-20 13:48:401940 const uint32_t kMinEncBitrateKbps = 100;
1941 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051942 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 13:48:401943 /*frame_size_pixels=*/codec_width_ * codec_height_,
1944 /*min_start_bitrate_bps=*/0,
1945 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1946 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051947 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1948
Sergey Silkincd02eba2020-01-20 13:48:401949 VideoEncoderConfig video_encoder_config;
1950 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1951 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1952 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1953 (kMinEncBitrateKbps + 1) * 1000;
1954 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1955 kMaxPayloadLength);
1956
1957 // When both encoder and app provide bitrate limits, the intersection of
1958 // provided sets should be used.
1959 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1960 WaitForEncodedFrame(1);
1961 EXPECT_EQ(kMaxEncBitrateKbps,
1962 bitrate_allocator_factory_.codec_config().maxBitrate);
1963 EXPECT_EQ(kMinEncBitrateKbps + 1,
1964 bitrate_allocator_factory_.codec_config().minBitrate);
1965
1966 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1967 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1968 (kMinEncBitrateKbps - 1) * 1000;
1969 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1970 kMaxPayloadLength);
1971 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 14:04:051972 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 13:48:401973 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 14:04:051974 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:401975 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051976 bitrate_allocator_factory_.codec_config().minBitrate);
1977
Sergey Silkincd02eba2020-01-20 13:48:401978 video_stream_encoder_->Stop();
1979}
1980
1981TEST_F(VideoStreamEncoderTest,
1982 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 16:49:071983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111984 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 13:48:401985
1986 const uint32_t kMinAppBitrateKbps = 100;
1987 const uint32_t kMaxAppBitrateKbps = 200;
1988 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1989 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1990 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1991 /*frame_size_pixels=*/codec_width_ * codec_height_,
1992 /*min_start_bitrate_bps=*/0,
1993 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1994 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1995 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1996
1997 VideoEncoderConfig video_encoder_config;
1998 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1999 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2000 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2001 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 14:04:052002 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2003 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 14:04:052004
Sergey Silkincd02eba2020-01-20 13:48:402005 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2006 WaitForEncodedFrame(1);
2007 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:052008 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:402009 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:052010 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 15:56:402011
2012 video_stream_encoder_->Stop();
2013}
2014
2015TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 14:04:052016 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 16:49:072017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112018 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:402019
2020 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 14:04:052021 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 15:56:402022 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 14:04:052023 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 15:56:402024 fake_encoder_.SetResolutionBitrateLimits(
2025 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2026
2027 VideoEncoderConfig video_encoder_config;
2028 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2029 video_encoder_config.max_bitrate_bps = 0;
2030 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2031 kMaxPayloadLength);
2032
Sergey Silkin6b2cec12019-08-09 14:04:052033 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 15:56:402034 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2035 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 14:04:052036 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2037 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402038 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2039 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2040
Sergey Silkin6b2cec12019-08-09 14:04:052041 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 15:56:402042 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2043 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 14:04:052044 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2045 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402046 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2047 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2048
Sergey Silkin6b2cec12019-08-09 14:04:052049 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 15:56:402050 // encoder for 360p should be used.
2051 video_source_.IncomingCapturedFrame(
2052 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2053 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 14:04:052054 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2055 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402056 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2057 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2058
Sergey Silkin6b2cec12019-08-09 14:04:052059 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 15:56:402060 // ignored.
2061 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2062 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 14:04:052063 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2064 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402065 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2066 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:052067 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2068 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402069 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2070 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2071
2072 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2073 // for 270p should be used.
2074 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2075 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 14:04:052076 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2077 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402078 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2079 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2080
2081 video_stream_encoder_->Stop();
2082}
2083
Sergey Silkin6b2cec12019-08-09 14:04:052084TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 16:49:072085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112086 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 14:04:052087
2088 VideoEncoderConfig video_encoder_config;
2089 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2090 video_encoder_config.max_bitrate_bps = 0;
2091 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2092 kMaxPayloadLength);
2093
2094 // Encode 720p frame to get the default encoder target bitrate.
2095 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2096 WaitForEncodedFrame(1);
2097 const uint32_t kDefaultTargetBitrateFor720pKbps =
2098 bitrate_allocator_factory_.codec_config()
2099 .simulcastStream[0]
2100 .targetBitrate;
2101
2102 // Set the max recommended encoder bitrate to something lower than the default
2103 // target bitrate.
2104 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2105 1280 * 720, 10 * 1000, 10 * 1000,
2106 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2107 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2108
2109 // Change resolution to trigger encoder reinitialization.
2110 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2111 WaitForEncodedFrame(2);
2112 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2113 WaitForEncodedFrame(3);
2114
2115 // Ensure the target bitrate is capped by the max bitrate.
2116 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2117 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2118 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2119 .simulcastStream[0]
2120 .targetBitrate *
2121 1000,
2122 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2123
2124 video_stream_encoder_->Stop();
2125}
2126
Åsa Perssona7e34d32021-01-20 14:36:132127TEST_F(VideoStreamEncoderTest,
2128 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2129 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2130 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2131 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2132 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2133 fake_encoder_.SetResolutionBitrateLimits(
2134 {kEncoderLimits270p, kEncoderLimits360p});
2135
2136 // Two streams, highest stream active.
2137 VideoEncoderConfig config;
2138 const int kNumStreams = 2;
2139 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2140 config.max_bitrate_bps = 0;
2141 config.simulcast_layers[0].active = false;
2142 config.simulcast_layers[1].active = true;
2143 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432144 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132145 "VP8", /*max qp*/ 56, /*screencast*/ false,
2146 /*screenshare enabled*/ false);
2147 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2148
2149 // The encoder bitrate limits for 270p should be used.
2150 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2151 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112152 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132153 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112154 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132155 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112156 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132157
2158 // The encoder bitrate limits for 360p should be used.
2159 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2160 EXPECT_FALSE(WaitForFrame(1000));
2161 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112162 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132163 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112164 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132165
2166 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2167 video_source_.IncomingCapturedFrame(
2168 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2169 EXPECT_FALSE(WaitForFrame(1000));
2170 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112171 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132172 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112173 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132174
2175 // Resolution higher than 360p. Encoder limits should be ignored.
2176 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2177 EXPECT_FALSE(WaitForFrame(1000));
2178 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112179 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132180 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112181 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132182 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112183 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132184 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112185 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132186
2187 // Resolution lower than 270p. The encoder limits for 270p should be used.
2188 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2189 EXPECT_FALSE(WaitForFrame(1000));
2190 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112191 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132192 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112193 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132194
2195 video_stream_encoder_->Stop();
2196}
2197
2198TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:512199 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2200 // Two streams, highest stream active.
2201 VideoEncoderConfig config;
2202 const int kNumStreams = 2;
2203 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2204 config.max_bitrate_bps = 0;
2205 config.simulcast_layers[0].active = false;
2206 config.simulcast_layers[1].active = true;
2207 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432208 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 09:39:512209 "VP8", /*max qp*/ 56, /*screencast*/ false,
2210 /*screenshare enabled*/ false);
2211 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2212
2213 // Default bitrate limits for 270p should be used.
2214 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2215 kDefaultLimits270p =
2216 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192217 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:512218 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2219 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112220 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 09:39:512221 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112222 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512223 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112224 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512225
2226 // Default bitrate limits for 360p should be used.
2227 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2228 kDefaultLimits360p =
2229 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192230 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:512231 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2232 EXPECT_FALSE(WaitForFrame(1000));
2233 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112234 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512235 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112236 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512237
2238 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2239 video_source_.IncomingCapturedFrame(
2240 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2241 EXPECT_FALSE(WaitForFrame(1000));
2242 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112243 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512244 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112245 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512246
2247 // Default bitrate limits for 540p should be used.
2248 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2249 kDefaultLimits540p =
2250 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192251 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 09:39:512252 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2253 EXPECT_FALSE(WaitForFrame(1000));
2254 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112255 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512256 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112257 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512258
2259 video_stream_encoder_->Stop();
2260}
2261
2262TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:132263 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2264 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2265 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2266 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2267 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2268 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2269 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2270 fake_encoder_.SetResolutionBitrateLimits(
2271 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2272
2273 // Three streams, middle stream active.
2274 VideoEncoderConfig config;
2275 const int kNumStreams = 3;
2276 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2277 config.simulcast_layers[0].active = false;
2278 config.simulcast_layers[1].active = true;
2279 config.simulcast_layers[2].active = false;
2280 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432281 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132282 "VP8", /*max qp*/ 56, /*screencast*/ false,
2283 /*screenshare enabled*/ false);
2284 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2285
2286 // The encoder bitrate limits for 360p should be used.
2287 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2288 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112289 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132290 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112291 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132292 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112293 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132294
2295 // The encoder bitrate limits for 270p should be used.
2296 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2297 EXPECT_FALSE(WaitForFrame(1000));
2298 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112299 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132300 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112301 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132302
2303 video_stream_encoder_->Stop();
2304}
2305
2306TEST_F(VideoStreamEncoderTest,
2307 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2308 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2309 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2310 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2311 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2312 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2313 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2314 fake_encoder_.SetResolutionBitrateLimits(
2315 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2316
2317 // Three streams, lowest stream active.
2318 VideoEncoderConfig config;
2319 const int kNumStreams = 3;
2320 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2321 config.simulcast_layers[0].active = true;
2322 config.simulcast_layers[1].active = false;
2323 config.simulcast_layers[2].active = false;
2324 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432325 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132326 "VP8", /*max qp*/ 56, /*screencast*/ false,
2327 /*screenshare enabled*/ false);
2328 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2329
2330 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2331 // on lowest stream, limits for 270p should not be used
2332 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2333 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112334 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132335 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112336 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132337 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112338 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132339
2340 video_stream_encoder_->Stop();
2341}
2342
2343TEST_F(VideoStreamEncoderTest,
2344 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2345 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2346 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2347 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2348 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2349 fake_encoder_.SetResolutionBitrateLimits(
2350 {kEncoderLimits270p, kEncoderLimits360p});
2351 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2352
2353 // Two streams, highest stream active.
2354 VideoEncoderConfig config;
2355 const int kNumStreams = 2;
2356 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2357 config.simulcast_layers[0].active = false;
2358 config.simulcast_layers[1].active = true;
2359 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2360 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432361 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132362 "VP8", /*max qp*/ 56, /*screencast*/ false,
2363 /*screenshare enabled*/ false);
2364 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2365
2366 // The encoder bitrate limits for 270p should be used.
2367 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2368 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112369 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132370 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112371 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132372 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112373 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132374
2375 // The max configured bitrate is less than the encoder limit for 360p.
2376 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2377 EXPECT_FALSE(WaitForFrame(1000));
2378 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112379 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132380 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 08:07:112381 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132382
2383 video_stream_encoder_->Stop();
2384}
2385
mflodmancc3d4422017-08-03 15:27:512386TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 18:45:462387 EXPECT_TRUE(video_source_.has_sinks());
2388 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512389 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412390 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:462391 EXPECT_FALSE(video_source_.has_sinks());
2392 EXPECT_TRUE(new_video_source.has_sinks());
2393
mflodmancc3d4422017-08-03 15:27:512394 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462395}
2396
mflodmancc3d4422017-08-03 15:27:512397TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 18:45:462398 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512399 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 18:45:462400 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512401 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462402}
2403
Åsa Perssonc5a74ff2020-09-20 15:50:002404class ResolutionAlignmentTest
2405 : public VideoStreamEncoderTest,
2406 public ::testing::WithParamInterface<
2407 ::testing::tuple<int, std::vector<double>>> {
2408 public:
2409 ResolutionAlignmentTest()
2410 : requested_alignment_(::testing::get<0>(GetParam())),
2411 scale_factors_(::testing::get<1>(GetParam())) {}
2412
2413 protected:
2414 const int requested_alignment_;
2415 const std::vector<double> scale_factors_;
2416};
2417
2418INSTANTIATE_TEST_SUITE_P(
2419 AlignmentAndScaleFactors,
2420 ResolutionAlignmentTest,
2421 ::testing::Combine(
2422 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2423 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2424 std::vector<double>{-1.0, -1.0},
2425 std::vector<double>{-1.0, -1.0, -1.0},
2426 std::vector<double>{4.0, 2.0, 1.0},
2427 std::vector<double>{9999.0, -1.0, 1.0},
2428 std::vector<double>{3.99, 2.01, 1.0},
2429 std::vector<double>{4.9, 1.7, 1.25},
2430 std::vector<double>{10.0, 4.0, 3.0},
2431 std::vector<double>{1.75, 3.5},
2432 std::vector<double>{1.5, 2.5},
2433 std::vector<double>{1.3, 1.0})));
2434
2435TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2436 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 08:47:112437 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 15:50:002438 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2439 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2440
2441 // Fill config with the scaling factor by which to reduce encoding size.
2442 const int num_streams = scale_factors_.size();
2443 VideoEncoderConfig config;
2444 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2445 for (int i = 0; i < num_streams; ++i) {
2446 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2447 }
2448 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432449 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 15:50:002450 "VP8", /*max qp*/ 56, /*screencast*/ false,
2451 /*screenshare enabled*/ false);
2452 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2453
Henrik Boström381d1092020-05-12 16:49:072454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112455 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2456 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 15:50:002457 // Wait for all layers before triggering event.
2458 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 08:47:112459
2460 // On the 1st frame, we should have initialized the encoder and
2461 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 15:50:002462 int64_t timestamp_ms = kFrameIntervalMs;
2463 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2464 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:112465 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 08:47:112466
2467 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2468 // (It's up the to the encoder to potentially drop the previous frame,
2469 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 15:50:002470 timestamp_ms += kFrameIntervalMs;
2471 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2472 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:112473 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 15:50:002474
Asa Persson606d3cb2021-10-04 08:07:112475 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 15:50:002476 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2477 // Frame size should be a multiple of the requested alignment.
2478 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2479 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2480 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2481 // Aspect ratio should match.
2482 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2483 codec.height * codec.simulcastStream[i].width);
2484 }
Rasmus Brandt5cad55b2019-12-19 08:47:112485
2486 video_stream_encoder_->Stop();
2487}
2488
Jonathan Yubc771b72017-12-09 01:04:292489TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2490 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-14 06:25:222491 const int kWidth = 1280;
2492 const int kHeight = 720;
Jonathan Yubc771b72017-12-09 01:04:292493
2494 // We rely on the automatic resolution adaptation, but we handle framerate
2495 // adaptation manually by mocking the stats proxy.
2496 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-14 06:25:222497
Taylor Brandstetter49fcc102018-05-16 21:20:412498 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 16:49:072499 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112500 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 21:20:412501 video_stream_encoder_->SetSource(&video_source_,
2502 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:392503 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:222504 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292505 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:222506 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2507
Jonathan Yubc771b72017-12-09 01:04:292508 // Adapt down as far as possible.
2509 rtc::VideoSinkWants last_wants;
2510 int64_t t = 1;
2511 int loop_count = 0;
2512 do {
2513 ++loop_count;
2514 last_wants = video_source_.sink_wants();
2515
2516 // Simulate the framerate we've been asked to adapt to.
2517 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2518 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2519 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2520 mock_stats.input_frame_rate = fps;
2521 stats_proxy_->SetMockStats(mock_stats);
2522
2523 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2524 sink_.WaitForEncodedFrame(t);
2525 t += frame_interval_ms;
2526
mflodmancc3d4422017-08-03 15:27:512527 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392528 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292529 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392530 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2531 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292532 } while (video_source_.sink_wants().max_pixel_count <
2533 last_wants.max_pixel_count ||
2534 video_source_.sink_wants().max_framerate_fps <
2535 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222536
Jonathan Yubc771b72017-12-09 01:04:292537 // Verify that we've adapted all the way down.
2538 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222539 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292540 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2541 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-14 06:25:222542 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:292543 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2544 *video_source_.last_sent_height());
2545 EXPECT_EQ(kMinBalancedFramerateFps,
2546 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222547
Jonathan Yubc771b72017-12-09 01:04:292548 // Adapt back up the same number of times we adapted down.
2549 for (int i = 0; i < loop_count - 1; ++i) {
2550 last_wants = video_source_.sink_wants();
2551
2552 // Simulate the framerate we've been asked to adapt to.
2553 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2554 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2555 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2556 mock_stats.input_frame_rate = fps;
2557 stats_proxy_->SetMockStats(mock_stats);
2558
2559 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2560 sink_.WaitForEncodedFrame(t);
2561 t += frame_interval_ms;
2562
Henrik Boström91aa7322020-04-28 10:24:332563 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392564 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292565 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392566 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2567 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292568 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2569 last_wants.max_pixel_count ||
2570 video_source_.sink_wants().max_framerate_fps >
2571 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222572 }
2573
Evan Shrubsole5cd7eb82020-05-25 12:08:392574 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-09 01:04:292575 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222576 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292577 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2578 EXPECT_EQ((loop_count - 1) * 2,
2579 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:222580
mflodmancc3d4422017-08-03 15:27:512581 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222582}
Rasmus Brandt5cad55b2019-12-19 08:47:112583
Evan Shrubsole2e2f6742020-05-14 08:41:152584TEST_F(VideoStreamEncoderTest,
2585 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 08:07:112586 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2587 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392588 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152589
2590 const int kFrameWidth = 1280;
2591 const int kFrameHeight = 720;
2592
2593 int64_t ntp_time = kFrameIntervalMs;
2594
2595 // Force an input frame rate to be available, or the adaptation call won't
2596 // know what framerate to adapt form.
2597 const int kInputFps = 30;
2598 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2599 stats.input_frame_rate = kInputFps;
2600 stats_proxy_->SetMockStats(stats);
2601
2602 video_source_.set_adaptation_enabled(true);
2603 video_stream_encoder_->SetSource(
2604 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 12:08:392605 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152606 video_source_.IncomingCapturedFrame(
2607 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2608 sink_.WaitForEncodedFrame(ntp_time);
2609 ntp_time += kFrameIntervalMs;
2610
2611 // Trigger CPU overuse.
2612 video_stream_encoder_->TriggerCpuOveruse();
2613 video_source_.IncomingCapturedFrame(
2614 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2615 sink_.WaitForEncodedFrame(ntp_time);
2616 ntp_time += kFrameIntervalMs;
2617
2618 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2619 EXPECT_EQ(std::numeric_limits<int>::max(),
2620 video_source_.sink_wants().max_pixel_count);
2621 // Some framerate constraint should be set.
2622 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2623 EXPECT_LT(restricted_fps, kInputFps);
2624 video_source_.IncomingCapturedFrame(
2625 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2626 sink_.WaitForEncodedFrame(ntp_time);
2627 ntp_time += 100;
2628
Henrik Boström2671dac2020-05-19 14:29:092629 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152630 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2631 // Give the encoder queue time to process the change in degradation preference
2632 // by waiting for an encoded frame.
2633 video_source_.IncomingCapturedFrame(
2634 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2635 sink_.WaitForEncodedFrame(ntp_time);
2636 ntp_time += kFrameIntervalMs;
2637
2638 video_stream_encoder_->TriggerQualityLow();
2639 video_source_.IncomingCapturedFrame(
2640 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2641 sink_.WaitForEncodedFrame(ntp_time);
2642 ntp_time += kFrameIntervalMs;
2643
2644 // Some resolution constraint should be set.
2645 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2646 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2647 kFrameWidth * kFrameHeight);
2648 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2649
2650 int pixel_count = video_source_.sink_wants().max_pixel_count;
2651 // Triggering a CPU underuse should not change the sink wants since it has
2652 // not been overused for resolution since we changed degradation preference.
2653 video_stream_encoder_->TriggerCpuUnderuse();
2654 video_source_.IncomingCapturedFrame(
2655 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2656 sink_.WaitForEncodedFrame(ntp_time);
2657 ntp_time += kFrameIntervalMs;
2658 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2659 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2660
Evan Shrubsole64469032020-06-11 08:45:292661 // Change the degradation preference back. CPU underuse should not adapt since
2662 // QP is most limited.
Henrik Boström2671dac2020-05-19 14:29:092663 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152664 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2665 video_source_.IncomingCapturedFrame(
2666 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2667 sink_.WaitForEncodedFrame(ntp_time);
2668 ntp_time += 100;
2669 // Resolution adaptations is gone after changing degradation preference.
2670 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2671 EXPECT_EQ(std::numeric_limits<int>::max(),
2672 video_source_.sink_wants().max_pixel_count);
2673 // The fps adaptation from above is now back.
2674 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2675
2676 // Trigger CPU underuse.
2677 video_stream_encoder_->TriggerCpuUnderuse();
2678 video_source_.IncomingCapturedFrame(
2679 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2680 sink_.WaitForEncodedFrame(ntp_time);
2681 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 08:45:292682 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2683
2684 // Trigger QP underuse, fps should return to normal.
2685 video_stream_encoder_->TriggerQualityHigh();
2686 video_source_.IncomingCapturedFrame(
2687 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2688 sink_.WaitForEncodedFrame(ntp_time);
2689 ntp_time += kFrameIntervalMs;
2690 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 08:41:152691
2692 video_stream_encoder_->Stop();
2693}
2694
mflodmancc3d4422017-08-03 15:27:512695TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 16:49:072696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112697 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392698 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 18:45:462699
sprangc5d62e22017-04-03 06:53:042700 const int kFrameWidth = 1280;
2701 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:042702
Åsa Persson8c1bf952018-09-13 08:42:192703 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 18:45:462704
kthelgason5e13d412016-12-01 11:59:512705 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042706 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522707 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042708 frame_timestamp += kFrameIntervalMs;
2709
perkj803d97f2016-11-01 18:45:462710 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512711 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462712 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042713 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522714 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042715 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 14:23:482716
asapersson0944a802017-04-07 07:57:582717 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-03 06:53:042718 // wanted resolution.
2719 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2720 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2721 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 08:42:192722 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042723
2724 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 18:45:462725 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 16:49:072726 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412727 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122728 // Give the encoder queue time to process the change in degradation preference
2729 // by waiting for an encoded frame.
2730 new_video_source.IncomingCapturedFrame(
2731 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2732 sink_.WaitForEncodedFrame(frame_timestamp);
2733 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042734 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 12:08:392735 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462736
sprangc5d62e22017-04-03 06:53:042737 // Force an input frame rate to be available, or the adaptation call won't
2738 // know what framerate to adapt form.
asapersson02465b82017-04-10 08:12:522739 const int kInputFps = 30;
sprangc5d62e22017-04-03 06:53:042740 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 08:12:522741 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-03 06:53:042742 stats_proxy_->SetMockStats(stats);
2743
mflodmancc3d4422017-08-03 15:27:512744 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462745 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042746 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522747 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042748 frame_timestamp += kFrameIntervalMs;
2749
2750 // Some framerate constraint should be set.
sprang84a37592017-02-10 15:04:272751 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-03 06:53:042752 EXPECT_EQ(std::numeric_limits<int>::max(),
2753 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522754 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-03 06:53:042755
asapersson02465b82017-04-10 08:12:522756 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 16:49:072757 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2758 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 10:42:122759 // Give the encoder queue time to process the change in degradation preference
2760 // by waiting for an encoded frame.
2761 new_video_source.IncomingCapturedFrame(
2762 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2763 sink_.WaitForEncodedFrame(frame_timestamp);
2764 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 12:08:392765 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-03 06:53:042766
mflodmancc3d4422017-08-03 15:27:512767 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042768 new_video_source.IncomingCapturedFrame(
2769 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522770 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042771 frame_timestamp += kFrameIntervalMs;
2772
2773 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 12:08:392774 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462775
2776 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072777 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412778 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 10:42:122779 // Give the encoder queue time to process the change in degradation preference
2780 // by waiting for an encoded frame.
2781 new_video_source.IncomingCapturedFrame(
2782 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2783 sink_.WaitForEncodedFrame(frame_timestamp);
2784 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042785 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2786 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 15:04:272787 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 08:42:192788 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042789
2790 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072791 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412792 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122793 // Give the encoder queue time to process the change in degradation preference
2794 // by waiting for an encoded frame.
2795 new_video_source.IncomingCapturedFrame(
2796 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2797 sink_.WaitForEncodedFrame(frame_timestamp);
2798 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042799 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2800 EXPECT_EQ(std::numeric_limits<int>::max(),
2801 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522802 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 18:45:462803
mflodmancc3d4422017-08-03 15:27:512804 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462805}
2806
mflodmancc3d4422017-08-03 15:27:512807TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112809 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 18:45:462810
asaperssonfab67072017-04-04 12:51:492811 const int kWidth = 1280;
2812 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492813 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522814 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 12:51:492815 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2816 EXPECT_FALSE(stats.bw_limited_resolution);
2817 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2818
2819 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512820 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492821 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522822 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 12:51:492823
2824 stats = stats_proxy_->GetStats();
2825 EXPECT_TRUE(stats.bw_limited_resolution);
2826 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2827
2828 // Trigger adapt up.
mflodmancc3d4422017-08-03 15:27:512829 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:492830 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522831 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 12:51:492832
2833 stats = stats_proxy_->GetStats();
2834 EXPECT_FALSE(stats.bw_limited_resolution);
2835 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2836 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2837
mflodmancc3d4422017-08-03 15:27:512838 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 12:51:492839}
2840
mflodmancc3d4422017-08-03 15:27:512841TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072842 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112843 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 12:51:492844
2845 const int kWidth = 1280;
2846 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492847 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522848 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 18:45:462849 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2850 EXPECT_FALSE(stats.cpu_limited_resolution);
2851 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2852
2853 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512854 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492855 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522856 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 18:45:462857
2858 stats = stats_proxy_->GetStats();
2859 EXPECT_TRUE(stats.cpu_limited_resolution);
2860 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2861
2862 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332863 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492864 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522865 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 18:45:462866
2867 stats = stats_proxy_->GetStats();
2868 EXPECT_FALSE(stats.cpu_limited_resolution);
2869 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 12:51:492870 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:462871
mflodmancc3d4422017-08-03 15:27:512872 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462873}
2874
mflodmancc3d4422017-08-03 15:27:512875TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072876 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112877 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112878
asaperssonfab67072017-04-04 12:51:492879 const int kWidth = 1280;
2880 const int kHeight = 720;
2881 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522882 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112883 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182884 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112885 EXPECT_FALSE(stats.cpu_limited_resolution);
2886 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2887
asaperssonfab67072017-04-04 12:51:492888 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512889 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492890 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522891 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112892 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182893 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112894 EXPECT_TRUE(stats.cpu_limited_resolution);
2895 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2896
2897 // Set new source with adaptation still enabled.
2898 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512899 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412900 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112901
asaperssonfab67072017-04-04 12:51:492902 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522903 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112904 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182905 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112906 EXPECT_TRUE(stats.cpu_limited_resolution);
2907 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2908
2909 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412910 video_stream_encoder_->SetSource(&new_video_source,
2911 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 09:44:112912
asaperssonfab67072017-04-04 12:51:492913 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522914 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112915 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182916 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112917 EXPECT_FALSE(stats.cpu_limited_resolution);
2918 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2919
2920 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 15:27:512921 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412922 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112923
asaperssonfab67072017-04-04 12:51:492924 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522925 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112926 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182927 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112928 EXPECT_TRUE(stats.cpu_limited_resolution);
2929 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2930
asaperssonfab67072017-04-04 12:51:492931 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332932 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492933 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522934 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 09:44:112935 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182936 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112937 EXPECT_FALSE(stats.cpu_limited_resolution);
2938 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:522939 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112940
mflodmancc3d4422017-08-03 15:27:512941 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112942}
2943
mflodmancc3d4422017-08-03 15:27:512944TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112946 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112947
asaperssonfab67072017-04-04 12:51:492948 const int kWidth = 1280;
2949 const int kHeight = 720;
2950 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522951 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112952 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112953 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022954 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492955 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112956
2957 // Set new source with adaptation still enabled.
2958 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 21:20:412959 video_stream_encoder_->SetSource(&new_video_source,
2960 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112961
asaperssonfab67072017-04-04 12:51:492962 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522963 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112964 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112965 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022966 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492967 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112968
asaperssonfab67072017-04-04 12:51:492969 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512970 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492971 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522972 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112973 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112974 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022975 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492976 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112977
asaperssonfab67072017-04-04 12:51:492978 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412979 video_stream_encoder_->SetSource(&new_video_source,
2980 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112981
asaperssonfab67072017-04-04 12:51:492982 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522983 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112984 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112985 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022986 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492987 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112988
asapersson02465b82017-04-10 08:12:522989 // Disable resolution scaling.
mflodmancc3d4422017-08-03 15:27:512990 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412991 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:112992
asaperssonfab67072017-04-04 12:51:492993 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522994 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112995 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112996 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022997 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492998 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2999 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 09:44:113000
mflodmancc3d4422017-08-03 15:27:513001 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:113002}
3003
mflodmancc3d4422017-08-03 15:27:513004TEST_F(VideoStreamEncoderTest,
3005 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 16:49:073006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113007 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 12:29:123008
3009 const int kWidth = 1280;
3010 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 08:42:193011 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 12:29:123012 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 08:42:193013 video_source_.IncomingCapturedFrame(
3014 CreateFrame(timestamp_ms, kWidth, kHeight));
3015 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123016 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3017 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3018 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3019
3020 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:513021 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193022 timestamp_ms += kFrameIntervalMs;
3023 video_source_.IncomingCapturedFrame(
3024 CreateFrame(timestamp_ms, kWidth, kHeight));
3025 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123026 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3027 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3028 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3029
3030 // Trigger overuse.
mflodmancc3d4422017-08-03 15:27:513031 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193032 timestamp_ms += kFrameIntervalMs;
3033 video_source_.IncomingCapturedFrame(
3034 CreateFrame(timestamp_ms, kWidth, kHeight));
3035 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123036 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3037 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3038 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3039
Niels Möller4db138e2018-04-19 07:04:133040 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 12:29:123041 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:133042
3043 VideoEncoderConfig video_encoder_config;
3044 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3045 // Make format different, to force recreation of encoder.
3046 video_encoder_config.video_format.parameters["foo"] = "foo";
3047 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473048 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 08:42:193049 timestamp_ms += kFrameIntervalMs;
3050 video_source_.IncomingCapturedFrame(
3051 CreateFrame(timestamp_ms, kWidth, kHeight));
3052 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123053 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3054 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3055 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3056
mflodmancc3d4422017-08-03 15:27:513057 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 12:29:123058}
3059
mflodmancc3d4422017-08-03 15:27:513060TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 17:39:323061 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 16:49:073062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113063 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 17:39:323064
3065 const int kWidth = 1280;
3066 const int kHeight = 720;
3067 int sequence = 1;
3068
3069 // Enable BALANCED preference, no initial limitation.
3070 test::FrameForwarder source;
3071 video_stream_encoder_->SetSource(&source,
3072 webrtc::DegradationPreference::BALANCED);
3073 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3074 WaitForEncodedFrame(sequence++);
3075 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3076 EXPECT_FALSE(stats.cpu_limited_resolution);
3077 EXPECT_FALSE(stats.cpu_limited_framerate);
3078 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3079
3080 // Trigger CPU overuse, should now adapt down.
3081 video_stream_encoder_->TriggerCpuOveruse();
3082 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3083 WaitForEncodedFrame(sequence++);
3084 stats = stats_proxy_->GetStats();
3085 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3086
3087 // Set new degradation preference should clear restrictions since we changed
3088 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 14:54:213089 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 17:39:323090 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3091 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3092 WaitForEncodedFrame(sequence++);
3093 stats = stats_proxy_->GetStats();
3094 EXPECT_FALSE(stats.cpu_limited_resolution);
3095 EXPECT_FALSE(stats.cpu_limited_framerate);
3096 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3097
3098 // Force an input frame rate to be available, or the adaptation call won't
3099 // know what framerate to adapt from.
3100 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3101 mock_stats.input_frame_rate = 30;
3102 stats_proxy_->SetMockStats(mock_stats);
3103 video_stream_encoder_->TriggerCpuOveruse();
3104 stats_proxy_->ResetMockStats();
3105 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3106 WaitForEncodedFrame(sequence++);
3107
3108 // We have now adapted once.
3109 stats = stats_proxy_->GetStats();
3110 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3111
3112 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 14:54:213113 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3114 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 17:39:323115 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3116 WaitForEncodedFrame(sequence++);
3117 stats = stats_proxy_->GetStats();
3118 EXPECT_FALSE(stats.cpu_limited_resolution);
3119 EXPECT_FALSE(stats.cpu_limited_framerate);
3120 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3121
3122 video_stream_encoder_->Stop();
3123}
3124
3125TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513126 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 16:49:073127 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113128 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 18:45:463129
asapersson0944a802017-04-07 07:57:583130 const int kWidth = 1280;
3131 const int kHeight = 720;
sprang84a37592017-02-10 15:04:273132 int sequence = 1;
perkj803d97f2016-11-01 18:45:463133
asaperssonfab67072017-04-04 12:51:493134 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523135 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463136 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 15:04:273137 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023138 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 15:04:273139 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3140
asapersson02465b82017-04-10 08:12:523141 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 15:27:513142 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:493143 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523144 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 15:04:273145 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 18:45:463146 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023147 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463148 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3149
3150 // Set new source with adaptation still enabled.
3151 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513152 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413153 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:463154
3155 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493156 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523157 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463158 stats = stats_proxy_->GetStats();
3159 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183160 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463161 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3162
sprangc5d62e22017-04-03 06:53:043163 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 15:27:513164 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413165 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 18:45:463166 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493167 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523168 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463169 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-03 06:53:043170 // Not adapted at first.
perkj803d97f2016-11-01 18:45:463171 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183172 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463173 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3174
sprangc5d62e22017-04-03 06:53:043175 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-16 06:40:183176 // know what framerate to adapt from.
sprangc5d62e22017-04-03 06:53:043177 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3178 mock_stats.input_frame_rate = 30;
3179 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513180 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043181 stats_proxy_->ResetMockStats();
3182
3183 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493184 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523185 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043186
3187 // Framerate now adapted.
3188 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:183189 EXPECT_FALSE(stats.cpu_limited_resolution);
3190 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043191 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3192
3193 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 21:20:413194 video_stream_encoder_->SetSource(&new_video_source,
3195 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-03 06:53:043196 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493197 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523198 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043199
3200 stats = stats_proxy_->GetStats();
3201 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023202 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043203 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3204
3205 // Try to trigger overuse. Should not succeed.
3206 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513207 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043208 stats_proxy_->ResetMockStats();
3209
3210 stats = stats_proxy_->GetStats();
3211 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023212 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043213 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3214
3215 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 15:27:513216 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413217 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 12:51:493218 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523219 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463220 stats = stats_proxy_->GetStats();
3221 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023222 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043223 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 18:45:463224
3225 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333226 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:493227 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523228 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463229 stats = stats_proxy_->GetStats();
3230 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023231 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043232 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3233
3234 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 15:27:513235 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413236 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:043237 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493238 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523239 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043240 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 07:01:023241 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-03 06:53:043242 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023243 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043244 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3245
3246 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333247 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:043248 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493249 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523250 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043251 stats = stats_proxy_->GetStats();
3252 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023253 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043254 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:523255 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:463256
mflodmancc3d4422017-08-03 15:27:513257 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:463258}
3259
mflodmancc3d4422017-08-03 15:27:513260TEST_F(VideoStreamEncoderTest,
3261 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 12:51:493262 const int kWidth = 1280;
3263 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113265 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:113266
asaperssonfab67072017-04-04 12:51:493267 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 12:08:393268 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 09:44:113269
asaperssonfab67072017-04-04 12:51:493270 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523271 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:113272
asaperssonfab67072017-04-04 12:51:493273 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513274 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 11:59:513275
asaperssonfab67072017-04-04 12:51:493276 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523277 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 11:59:513278
kthelgason876222f2016-11-29 09:44:113279 // Expect a scale down.
3280 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 12:51:493281 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 09:44:113282
asapersson02465b82017-04-10 08:12:523283 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 09:44:113284 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513285 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413286 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:113287
asaperssonfab67072017-04-04 12:51:493288 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513289 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:493290 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523291 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:113292
asaperssonfab67072017-04-04 12:51:493293 // Expect no scaling.
sprangc5d62e22017-04-03 06:53:043294 EXPECT_EQ(std::numeric_limits<int>::max(),
3295 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113296
asaperssonfab67072017-04-04 12:51:493297 // Trigger scale up.
mflodmancc3d4422017-08-03 15:27:513298 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:493299 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523300 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:113301
asapersson02465b82017-04-10 08:12:523302 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-03 06:53:043303 EXPECT_EQ(std::numeric_limits<int>::max(),
3304 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113305
mflodmancc3d4422017-08-03 15:27:513306 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:113307}
3308
mflodmancc3d4422017-08-03 15:27:513309TEST_F(VideoStreamEncoderTest,
3310 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523311 const int kWidth = 1280;
3312 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113314 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523315
Taylor Brandstetter49fcc102018-05-16 21:20:413316 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523317 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513318 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413319 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523320
3321 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523322 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393323 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523324 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3325 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3326
3327 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513328 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393329 EXPECT_THAT(source.sink_wants(),
3330 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523331 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3332 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3333 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3334
3335 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 15:27:513336 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 08:12:523337 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3338 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3339 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3340
mflodmancc3d4422017-08-03 15:27:513341 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523342}
3343
mflodmancc3d4422017-08-03 15:27:513344TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223345 const int kWidth = 1280;
3346 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113348 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223349
Taylor Brandstetter49fcc102018-05-16 21:20:413350 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223351 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413352 video_stream_encoder_->SetSource(&source,
3353 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223354 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3355 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393356 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223357
3358 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513359 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:393360 EXPECT_THAT(source.sink_wants(),
3361 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223362 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3363 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3364 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3365
3366 // Trigger adapt down for same input resolution, expect no change.
3367 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3368 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:513369 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223370 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3371 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3372 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3373
3374 // Trigger adapt down for larger input resolution, expect no change.
3375 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3376 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:513377 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223378 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3380 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3381
mflodmancc3d4422017-08-03 15:27:513382 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223383}
3384
mflodmancc3d4422017-08-03 15:27:513385TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 20:55:383386 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3387 const int kWidth = 640;
3388 const int kHeight = 360;
3389 const int64_t kFrameIntervalMs = 150;
3390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113391 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 20:55:383392
3393 // Enable BALANCED preference, no initial limitation.
3394 AdaptingFrameForwarder source(&time_controller_);
3395 source.set_adaptation_enabled(true);
3396 video_stream_encoder_->SetSource(&source,
3397 webrtc::DegradationPreference::BALANCED);
3398
3399 int64_t timestamp_ms = kFrameIntervalMs;
3400 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3401 sink_.WaitForEncodedFrame(kWidth, kHeight);
3402 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3404 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3405 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3406
3407 // Trigger adapt down, expect reduced fps (640x360@15fps).
3408 video_stream_encoder_->TriggerQualityLow();
3409 timestamp_ms += kFrameIntervalMs;
3410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3411 sink_.WaitForEncodedFrame(timestamp_ms);
3412 EXPECT_THAT(source.sink_wants(),
3413 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3415 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3416 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3417
3418 // Source requests 270p, expect reduced resolution (480x270@15fps).
3419 source.OnOutputFormatRequest(480, 270);
3420 timestamp_ms += kFrameIntervalMs;
3421 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3422 WaitForEncodedFrame(480, 270);
3423 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3424
3425 // Trigger adapt down, expect reduced fps (480x270@10fps).
3426 video_stream_encoder_->TriggerQualityLow();
3427 timestamp_ms += kFrameIntervalMs;
3428 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3429 sink_.WaitForEncodedFrame(timestamp_ms);
3430 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3431 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3432 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3433 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3434
3435 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3436 source.OnOutputFormatRequest(320, 180);
3437 timestamp_ms += kFrameIntervalMs;
3438 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3439 WaitForEncodedFrame(320, 180);
3440 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3441
3442 // Trigger adapt down, expect reduced fps (320x180@7fps).
3443 video_stream_encoder_->TriggerQualityLow();
3444 timestamp_ms += kFrameIntervalMs;
3445 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3446 sink_.WaitForEncodedFrame(timestamp_ms);
3447 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3448 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3449 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3450 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3451
3452 // Source requests VGA, expect increased resolution (640x360@7fps).
3453 source.OnOutputFormatRequest(640, 360);
3454 timestamp_ms += kFrameIntervalMs;
3455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3456 WaitForEncodedFrame(timestamp_ms);
3457 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3458
3459 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3460 video_stream_encoder_->TriggerQualityHigh();
3461 timestamp_ms += kFrameIntervalMs;
3462 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3463 WaitForEncodedFrame(timestamp_ms);
3464 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3466 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3467 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3468
3469 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3470 video_stream_encoder_->TriggerQualityHigh();
3471 timestamp_ms += kFrameIntervalMs;
3472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3473 WaitForEncodedFrame(timestamp_ms);
3474 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3475 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3476 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3477 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3478
3479 // Trigger adapt up, expect increased fps (640x360@maxfps).
3480 video_stream_encoder_->TriggerQualityHigh();
3481 timestamp_ms += kFrameIntervalMs;
3482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3483 WaitForEncodedFrame(timestamp_ms);
3484 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3487 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3488
3489 video_stream_encoder_->Stop();
3490}
3491
3492TEST_F(VideoStreamEncoderTest,
3493 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3494 const int kWidth = 1280;
3495 const int kHeight = 720;
3496 const int64_t kFrameIntervalMs = 150;
3497 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113498 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 20:55:383499
3500 // Enable BALANCED preference, no initial limitation.
3501 AdaptingFrameForwarder source(&time_controller_);
3502 source.set_adaptation_enabled(true);
3503 video_stream_encoder_->SetSource(&source,
3504 webrtc::DegradationPreference::BALANCED);
3505
3506 int64_t timestamp_ms = kFrameIntervalMs;
3507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3508 sink_.WaitForEncodedFrame(kWidth, kHeight);
3509 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3510 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3511 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3512 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3513
3514 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3515 video_stream_encoder_->TriggerQualityLow();
3516 timestamp_ms += kFrameIntervalMs;
3517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3518 sink_.WaitForEncodedFrame(timestamp_ms);
3519 EXPECT_THAT(source.sink_wants(),
3520 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3521 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3522 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3523 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3524
3525 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3526 video_stream_encoder_->TriggerQualityLow();
3527 timestamp_ms += kFrameIntervalMs;
3528 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3529 sink_.WaitForEncodedFrame(timestamp_ms);
3530 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3531 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3532 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3533 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3534
3535 // Trigger adapt down, expect reduced fps (640x360@15fps).
3536 video_stream_encoder_->TriggerQualityLow();
3537 timestamp_ms += kFrameIntervalMs;
3538 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3539 WaitForEncodedFrame(timestamp_ms);
3540 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3541 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3542 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3543 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3544
3545 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3546 source.OnOutputFormatRequest(320, 180);
3547 timestamp_ms += kFrameIntervalMs;
3548 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3549 WaitForEncodedFrame(320, 180);
3550 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3551 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3552
3553 // Trigger adapt down, expect reduced fps (320x180@7fps).
3554 video_stream_encoder_->TriggerCpuOveruse();
3555 timestamp_ms += kFrameIntervalMs;
3556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3557 WaitForEncodedFrame(timestamp_ms);
3558 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3561 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3562 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3563 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3565
3566 // Source requests HD, expect increased resolution (640x360@7fps).
3567 source.OnOutputFormatRequest(1280, 720);
3568 timestamp_ms += kFrameIntervalMs;
3569 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3570 WaitForEncodedFrame(timestamp_ms);
3571 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3572 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3573
3574 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3575 video_stream_encoder_->TriggerCpuUnderuse();
3576 timestamp_ms += kFrameIntervalMs;
3577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3578 WaitForEncodedFrame(timestamp_ms);
3579 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3582 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3583 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3584 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3585 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3586
3587 // Trigger adapt up, expect increased fps (640x360@maxfps).
3588 video_stream_encoder_->TriggerQualityHigh();
3589 video_stream_encoder_->TriggerCpuUnderuse();
3590 timestamp_ms += kFrameIntervalMs;
3591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3592 WaitForEncodedFrame(timestamp_ms);
3593 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3594 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3595 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3596 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3597 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3598 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3599 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3600
3601 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3602 video_stream_encoder_->TriggerQualityHigh();
3603 video_stream_encoder_->TriggerCpuUnderuse();
3604 timestamp_ms += kFrameIntervalMs;
3605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3606 WaitForEncodedFrame(timestamp_ms);
3607 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3610 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3611 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3612 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3613 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3614
3615 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3616 video_stream_encoder_->TriggerQualityHigh();
3617 video_stream_encoder_->TriggerCpuUnderuse();
3618 timestamp_ms += kFrameIntervalMs;
3619 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3620 WaitForEncodedFrame(timestamp_ms);
3621 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3622 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3623 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3624 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3625 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3626 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3627 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3628
3629 video_stream_encoder_->Stop();
3630}
3631
3632TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513633 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523634 const int kWidth = 1280;
3635 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113637 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523638
Taylor Brandstetter49fcc102018-05-16 21:20:413639 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523640 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513641 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413642 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523643
3644 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523645 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393646 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523647 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3648 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3649
3650 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333651 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393652 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523653 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3654 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3655
mflodmancc3d4422017-08-03 15:27:513656 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523657}
3658
mflodmancc3d4422017-08-03 15:27:513659TEST_F(VideoStreamEncoderTest,
3660 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 08:12:523661 const int kWidth = 1280;
3662 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113664 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523665
Taylor Brandstetter49fcc102018-05-16 21:20:413666 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523667 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513668 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413669 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 08:12:523670
3671 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523672 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393673 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183674 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523675 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3676
3677 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333678 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393679 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183680 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523681 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3682
mflodmancc3d4422017-08-03 15:27:513683 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523684}
3685
mflodmancc3d4422017-08-03 15:27:513686TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223687 const int kWidth = 1280;
3688 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073689 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113690 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223691
Taylor Brandstetter49fcc102018-05-16 21:20:413692 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223693 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413694 video_stream_encoder_->SetSource(&source,
3695 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223696
3697 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3698 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393699 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223700 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3701 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3702 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3703
3704 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513705 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393706 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223707 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3708 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3709 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3710
mflodmancc3d4422017-08-03 15:27:513711 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223712}
3713
mflodmancc3d4422017-08-03 15:27:513714TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-16 06:40:183715 const int kWidth = 1280;
3716 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113718 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:183719
Taylor Brandstetter49fcc102018-05-16 21:20:413720 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-16 06:40:183721 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413722 video_stream_encoder_->SetSource(&source,
3723 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-16 06:40:183724
3725 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3726 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393727 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183728 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3730 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3731
3732 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513733 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393734 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183735 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3736 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3737 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3738
mflodmancc3d4422017-08-03 15:27:513739 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183740}
3741
mflodmancc3d4422017-08-03 15:27:513742TEST_F(VideoStreamEncoderTest,
3743 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523744 const int kWidth = 1280;
3745 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073746 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113747 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523748
Taylor Brandstetter49fcc102018-05-16 21:20:413749 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233750 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 08:12:523751 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513752 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413753 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523754
3755 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523756 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393757 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523758 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3759 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3760
3761 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513762 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 08:12:523763 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523764 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393765 EXPECT_THAT(source.sink_wants(),
3766 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523767 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3768 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3769
3770 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513771 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393772 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523773 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3774 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3775 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3776
mflodmancc3d4422017-08-03 15:27:513777 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523778}
3779
mflodmancc3d4422017-08-03 15:27:513780TEST_F(VideoStreamEncoderTest,
3781 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-16 06:40:183782 const int kWidth = 1280;
3783 const int kHeight = 720;
3784 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 16:49:073785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113786 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:183787
3788 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3789 stats.input_frame_rate = kInputFps;
3790 stats_proxy_->SetMockStats(stats);
3791
Taylor Brandstetter49fcc102018-05-16 21:20:413792 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-16 06:40:183793 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3794 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393795 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183796
3797 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513798 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-16 06:40:183799 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3800 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393801 EXPECT_THAT(video_source_.sink_wants(),
3802 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-16 06:40:183803
Taylor Brandstetter49fcc102018-05-16 21:20:413804 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-16 06:40:183805 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 14:29:093806 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:413807 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:123808 // Give the encoder queue time to process the change in degradation preference
3809 // by waiting for an encoded frame.
3810 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3811 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 12:08:393812 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183813
3814 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 15:27:513815 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 10:42:123816 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3817 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 12:08:393818 EXPECT_THAT(new_video_source.sink_wants(),
3819 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-16 06:40:183820
3821 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513822 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393823 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183824
mflodmancc3d4422017-08-03 15:27:513825 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183826}
3827
mflodmancc3d4422017-08-03 15:27:513828TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 08:47:313829 const int kWidth = 1280;
3830 const int kHeight = 720;
3831 const size_t kNumFrames = 10;
3832
Henrik Boström381d1092020-05-12 16:49:073833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113834 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 11:59:513835
asaperssond0de2952017-04-21 08:47:313836 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 15:58:543837 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 08:47:313838 video_source_.set_adaptation_enabled(true);
3839
3840 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3841 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3842
3843 int downscales = 0;
3844 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 08:42:193845 video_source_.IncomingCapturedFrame(
3846 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3847 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 08:47:313848
asaperssonfab67072017-04-04 12:51:493849 // Trigger scale down.
asaperssond0de2952017-04-21 08:47:313850 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 15:27:513851 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-03 06:53:043852 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 08:47:313853
3854 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3855 ++downscales;
3856
3857 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3858 EXPECT_EQ(downscales,
3859 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3860 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 11:59:513861 }
mflodmancc3d4422017-08-03 15:27:513862 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313863}
3864
mflodmancc3d4422017-08-03 15:27:513865TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:313866 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3867 const int kWidth = 1280;
3868 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113870 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:313871
Taylor Brandstetter49fcc102018-05-16 21:20:413872 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233873 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:313874 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513875 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413876 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:313877
Åsa Persson8c1bf952018-09-13 08:42:193878 int64_t timestamp_ms = kFrameIntervalMs;
3879 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523880 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393881 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313882 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3883 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3884
3885 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513886 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193887 timestamp_ms += kFrameIntervalMs;
3888 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3889 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393890 EXPECT_THAT(source.sink_wants(),
3891 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313892 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3893 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3894
3895 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333896 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193897 timestamp_ms += kFrameIntervalMs;
3898 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523899 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393900 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313901 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3902 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3903
3904 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513905 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193906 timestamp_ms += kFrameIntervalMs;
3907 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3908 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393909 EXPECT_THAT(source.sink_wants(),
3910 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313911 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3912 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3913
3914 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333915 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193916 timestamp_ms += kFrameIntervalMs;
3917 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-16 06:40:183918 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393919 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313920 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3921 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3922
mflodmancc3d4422017-08-03 15:27:513923 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313924}
3925
mflodmancc3d4422017-08-03 15:27:513926TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-14 06:25:223927 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3928 const int kWidth = 1280;
3929 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073930 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113931 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223932
Taylor Brandstetter49fcc102018-05-16 21:20:413933 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233934 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:223935 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:413936 video_stream_encoder_->SetSource(&source,
3937 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223938
Åsa Persson8c1bf952018-09-13 08:42:193939 int64_t timestamp_ms = kFrameIntervalMs;
3940 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223941 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393942 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223943 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3944 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3945
3946 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513947 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193948 timestamp_ms += kFrameIntervalMs;
3949 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3950 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393951 EXPECT_THAT(source.sink_wants(),
3952 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223953 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3954 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3955
3956 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513957 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193958 timestamp_ms += kFrameIntervalMs;
3959 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223960 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393961 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223962 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3963 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3964
3965 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513966 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193967 timestamp_ms += kFrameIntervalMs;
3968 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3969 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393970 EXPECT_THAT(source.sink_wants(),
3971 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223972 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3973 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3974
3975 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513976 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193977 timestamp_ms += kFrameIntervalMs;
3978 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223979 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393980 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223981 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3982 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3983
mflodmancc3d4422017-08-03 15:27:513984 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223985}
3986
Sergey Silkin41c650b2019-10-14 11:12:193987TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3988 fake_encoder_.SetResolutionBitrateLimits(
3989 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3990
Henrik Boström381d1092020-05-12 16:49:073991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073992 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3993 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3994 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3995 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:193996
3997 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233998 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:193999 source.set_adaptation_enabled(true);
4000 video_stream_encoder_->SetSource(
4001 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4002
4003 // Insert 720p frame.
4004 int64_t timestamp_ms = kFrameIntervalMs;
4005 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4006 WaitForEncodedFrame(1280, 720);
4007
4008 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 16:49:074009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074010 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4011 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4012 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4013 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194014 video_stream_encoder_->TriggerQualityLow();
4015
4016 // Insert 720p frame. It should be downscaled and encoded.
4017 timestamp_ms += kFrameIntervalMs;
4018 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4019 WaitForEncodedFrame(960, 540);
4020
4021 // Trigger adapt up. Higher resolution should not be requested duo to lack
4022 // of bitrate.
4023 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:394024 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 11:12:194025
4026 // Increase bitrate.
Henrik Boström381d1092020-05-12 16:49:074027 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074028 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4029 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4030 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4031 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194032
4033 // Trigger adapt up. Higher resolution should be requested.
4034 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:394035 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 11:12:194036
4037 video_stream_encoder_->Stop();
4038}
4039
4040TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4041 fake_encoder_.SetResolutionBitrateLimits(
4042 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4043
4044 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 16:49:074045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074046 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4047 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4048 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4049 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194050
4051 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234052 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:194053 source.set_adaptation_enabled(true);
4054 video_stream_encoder_->SetSource(
4055 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4056
4057 // Insert 720p frame. It should be dropped and lower resolution should be
4058 // requested.
4059 int64_t timestamp_ms = kFrameIntervalMs;
4060 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4061 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:094062 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 11:12:194063
4064 // Insert 720p frame. It should be downscaled and encoded.
4065 timestamp_ms += kFrameIntervalMs;
4066 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4067 WaitForEncodedFrame(960, 540);
4068
4069 video_stream_encoder_->Stop();
4070}
4071
Åsa Perssonb67c44c2019-09-24 13:25:324072class BalancedDegradationTest : public VideoStreamEncoderTest {
4073 protected:
4074 void SetupTest() {
4075 // Reset encoder for field trials to take effect.
4076 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:114077 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 13:25:324078
4079 // Enable BALANCED preference.
4080 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 13:13:044081 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4082 }
4083
Asa Persson606d3cb2021-10-04 08:07:114084 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 16:49:074085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114086 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 13:25:324087 }
4088
Åsa Persson45b176f2019-09-30 09:19:054089 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 13:25:324090 timestamp_ms_ += kFrameIntervalMs;
4091 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 09:19:054092 }
4093
4094 void InsertFrameAndWaitForEncoded() {
4095 InsertFrame();
Åsa Perssonb67c44c2019-09-24 13:25:324096 sink_.WaitForEncodedFrame(timestamp_ms_);
4097 }
4098
4099 const int kWidth = 640; // pixels:640x360=230400
4100 const int kHeight = 360;
4101 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4102 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:234103 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 13:25:324104};
4105
Evan Shrubsolea1c77f62020-08-10 09:01:064106TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 13:25:324107 test::ScopedFieldTrials field_trials(
4108 "WebRTC-Video-BalancedDegradationSettings/"
4109 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4110 SetupTest();
4111
4112 // Force input frame rate.
4113 const int kInputFps = 24;
4114 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4115 stats.input_frame_rate = kInputFps;
4116 stats_proxy_->SetMockStats(stats);
4117
Åsa Persson45b176f2019-09-30 09:19:054118 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394119 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324120
Evan Shrubsolea1c77f62020-08-10 09:01:064121 // Trigger adapt down, expect scaled down framerate and resolution,
4122 // since Fps diff (input-requested:0) < threshold.
4123 video_stream_encoder_->TriggerQualityLow();
4124 EXPECT_THAT(source_.sink_wants(),
4125 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 13:25:324126
4127 video_stream_encoder_->Stop();
4128}
4129
Evan Shrubsolea1c77f62020-08-10 09:01:064130TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 13:25:324131 test::ScopedFieldTrials field_trials(
4132 "WebRTC-Video-BalancedDegradationSettings/"
4133 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4134 SetupTest();
4135
4136 // Force input frame rate.
4137 const int kInputFps = 25;
4138 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4139 stats.input_frame_rate = kInputFps;
4140 stats_proxy_->SetMockStats(stats);
4141
Åsa Persson45b176f2019-09-30 09:19:054142 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394143 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324144
Evan Shrubsolea1c77f62020-08-10 09:01:064145 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4146 // Fps diff (input-requested:1) == threshold.
4147 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:394148 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 13:25:324149
4150 video_stream_encoder_->Stop();
4151}
4152
4153TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4154 test::ScopedFieldTrials field_trials(
4155 "WebRTC-Video-BalancedDegradationSettings/"
4156 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4157 SetupTest();
4158
4159 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4160
Åsa Persson45b176f2019-09-30 09:19:054161 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394162 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324163
4164 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4165 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:394166 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 13:25:324167
4168 video_stream_encoder_->Stop();
4169}
4170
Åsa Perssonccfb3402019-09-25 13:13:044171TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324172 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 15:26:394173 "WebRTC-Video-BalancedDegradationSettings/"
4174 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 13:13:044175 SetupTest();
Åsa Persson1b247f12019-08-14 15:26:394176
Asa Persson606d3cb2021-10-04 08:07:114177 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4178 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4179 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 15:26:394180
Åsa Persson45b176f2019-09-30 09:19:054181 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394182 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 15:26:394183 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4184
4185 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4186 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054187 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394188 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 15:26:394189 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4190
4191 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4192 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054193 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544194 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:394195 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4196
Åsa Persson30ab0152019-08-27 10:22:334197 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4198 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054199 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544200 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 13:13:044201 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 10:22:334202 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4203
4204 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 15:26:394205 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054206 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334207 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 15:26:394208
Åsa Persson30ab0152019-08-27 10:22:334209 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114210 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 15:26:394211 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054212 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:044213 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 10:22:334214 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4215
4216 video_stream_encoder_->Stop();
4217}
4218
Åsa Perssonccfb3402019-09-25 13:13:044219TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 09:19:054220 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4221 test::ScopedFieldTrials field_trials(
4222 "WebRTC-Video-BalancedDegradationSettings/"
4223 "pixels:57600|129600|230400,fps:7|24|24/");
4224 SetupTest();
Asa Persson606d3cb2021-10-04 08:07:114225 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 09:19:054226
Evan Shrubsole5cd7eb82020-05-25 12:08:394227 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 09:19:054228
4229 // Insert frame, expect scaled down:
4230 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4231 InsertFrame();
4232 EXPECT_FALSE(WaitForFrame(1000));
4233 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4234 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4235
4236 // Insert frame, expect scaled down:
4237 // resolution (320x180@24fps).
4238 InsertFrame();
4239 EXPECT_FALSE(WaitForFrame(1000));
4240 EXPECT_LT(source_.sink_wants().max_pixel_count,
4241 source_.last_wants().max_pixel_count);
4242 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4243
4244 // Frame should not be dropped (min pixels per frame reached).
4245 InsertFrameAndWaitForEncoded();
4246
4247 video_stream_encoder_->Stop();
4248}
4249
4250TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334251 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324252 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:334253 "WebRTC-Video-BalancedDegradationSettings/"
4254 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044255 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334256
Asa Persson606d3cb2021-10-04 08:07:114257 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4258 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4259 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 10:22:334260
Åsa Persson45b176f2019-09-30 09:19:054261 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394262 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334263 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4264
4265 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4266 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054267 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394268 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334269 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4270
4271 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4272 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054273 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544274 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334275 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4276
4277 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4278 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054279 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544280 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:394281 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4282
Åsa Persson30ab0152019-08-27 10:22:334283 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4284 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054285 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544286 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334287 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4288
4289 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4290 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054291 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334292 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4293
4294 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114295 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334296 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054297 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544298 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334299 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4300
4301 video_stream_encoder_->Stop();
4302}
4303
Åsa Perssonccfb3402019-09-25 13:13:044304TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334305 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324306 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:334307 "WebRTC-Video-BalancedDegradationSettings/"
4308 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044309 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334310
Asa Persson606d3cb2021-10-04 08:07:114311 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4312 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4313 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4314 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4315 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334316
Åsa Persson45b176f2019-09-30 09:19:054317 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394318 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334319 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4320
4321 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4322 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054323 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394324 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334325 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4326
4327 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4328 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054329 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544330 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334331 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4332
4333 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4334 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054335 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544336 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334337 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4338
4339 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4340 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054341 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334342 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4343
4344 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114345 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334346 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054347 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544348 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334349 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4350
4351 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114352 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 10:22:334353 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054354 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334355 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4356
4357 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114358 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334359 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054360 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544361 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334362 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4363
Åsa Persson1b247f12019-08-14 15:26:394364 video_stream_encoder_->Stop();
4365}
4366
mflodmancc3d4422017-08-03 15:27:514367TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:314368 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4369 const int kWidth = 1280;
4370 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:074371 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114372 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:314373
Taylor Brandstetter49fcc102018-05-16 21:20:414374 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234375 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:314376 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:514377 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414378 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:314379
Åsa Persson8c1bf952018-09-13 08:42:194380 int64_t timestamp_ms = kFrameIntervalMs;
4381 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524382 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:394383 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314384 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4385 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4386 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4387 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4388
4389 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 15:27:514390 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194391 timestamp_ms += kFrameIntervalMs;
4392 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4393 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:394394 EXPECT_THAT(source.sink_wants(),
4395 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:314396 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4397 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4398 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4399 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4400
4401 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 15:27:514402 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194403 timestamp_ms += kFrameIntervalMs;
4404 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4405 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544406 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314407 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4408 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4409 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4410 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4411
Jonathan Yubc771b72017-12-09 01:04:294412 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 15:27:514413 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194414 timestamp_ms += kFrameIntervalMs;
4415 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4416 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544417 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314418 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4419 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294420 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:314421 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4422
Jonathan Yubc771b72017-12-09 01:04:294423 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 15:27:514424 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194425 timestamp_ms += kFrameIntervalMs;
4426 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4427 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544428 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294429 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 08:47:314430 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4431 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4432 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4433 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4434
Jonathan Yubc771b72017-12-09 01:04:294435 // Trigger quality adapt down, expect no change (min resolution reached).
4436 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194437 timestamp_ms += kFrameIntervalMs;
4438 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4439 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544440 EXPECT_THAT(source.sink_wants(), FpsMax());
4441 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-09 01:04:294442 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4443 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4444 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4445 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4446
Evan Shrubsole64469032020-06-11 08:45:294447 // Trigger quality adapt up, expect upscaled resolution (480x270).
4448 video_stream_encoder_->TriggerQualityHigh();
4449 timestamp_ms += kFrameIntervalMs;
4450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4451 WaitForEncodedFrame(timestamp_ms);
4452 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4453 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4454 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4455 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4456 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4457
4458 // Trigger quality and cpu adapt up since both are most limited, expect
4459 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 10:24:334460 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294461 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194462 timestamp_ms += kFrameIntervalMs;
4463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4464 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544465 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294466 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4467 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4468 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294469 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:294470
Evan Shrubsole64469032020-06-11 08:45:294471 // Trigger quality and cpu adapt up since both are most limited, expect
4472 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 10:24:334473 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294474 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194475 timestamp_ms += kFrameIntervalMs;
4476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4477 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544478 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314479 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 08:45:294480 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 08:47:314481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:294482 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4483 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314484
Evan Shrubsole64469032020-06-11 08:45:294485 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4486 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 10:24:334487 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:194488 timestamp_ms += kFrameIntervalMs;
4489 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4490 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544491 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 08:47:314492 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4493 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294494 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294495 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314496
4497 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 15:27:514498 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194499 timestamp_ms += kFrameIntervalMs;
4500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524501 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:544502 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:394503 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314504 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4505 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294506 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294507 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 11:59:514508
mflodmancc3d4422017-08-03 15:27:514509 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 11:59:514510}
4511
mflodmancc3d4422017-08-03 15:27:514512TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 12:51:494513 const int kWidth = 640;
4514 const int kHeight = 360;
perkj803d97f2016-11-01 18:45:464515
Henrik Boström381d1092020-05-12 16:49:074516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114517 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:524518
perkj803d97f2016-11-01 18:45:464519 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494520 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524521 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 18:45:464522 }
4523
mflodmancc3d4422017-08-03 15:27:514524 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:464525 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494526 video_source_.IncomingCapturedFrame(CreateFrame(
4527 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524528 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 18:45:464529 }
4530
mflodmancc3d4422017-08-03 15:27:514531 video_stream_encoder_->Stop();
4532 video_stream_encoder_.reset();
perkj803d97f2016-11-01 18:45:464533 stats_proxy_.reset();
sprangf8ee65e2017-02-28 16:49:334534
Ying Wangef3998f2019-12-09 12:06:534535 EXPECT_METRIC_EQ(
4536 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4537 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 18:45:464538 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4539}
4540
mflodmancc3d4422017-08-03 15:27:514541TEST_F(VideoStreamEncoderTest,
4542 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 16:49:074543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114544 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 09:01:064545 const int kWidth = 640;
4546 const int kHeight = 360;
4547
Taylor Brandstetter49fcc102018-05-16 21:20:414548 video_stream_encoder_->SetSource(&video_source_,
4549 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 09:01:064550
4551 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4552 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524553 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 09:01:064554 }
4555
mflodmancc3d4422017-08-03 15:27:514556 video_stream_encoder_->Stop();
4557 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 09:01:064558 stats_proxy_.reset();
4559
4560 EXPECT_EQ(0,
4561 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4562}
4563
Per Kjellanderdcef6412020-10-07 13:09:054564TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4565 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034566 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:054567 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 14:24:024568
4569 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 10:32:224570 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 08:07:114571 SimulcastRateAllocator(fake_encoder_.config())
4572 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 13:16:284573 kDefaultFps));
sprang57c2fff2017-01-16 14:24:024574
Henrik Boström381d1092020-05-12 16:49:074575 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114576 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 14:24:024577
sprang57c2fff2017-01-16 14:24:024578 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234579 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4580 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054581 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4582 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4583
Erik Språngd7329ca2019-02-21 20:19:534584 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 15:44:424585 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 12:31:234586 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024587
Per Kjellanderdcef6412020-10-07 13:09:054588 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 14:24:024589 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234590 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4591 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054592 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 12:31:234593 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024594
Per Kjellanderdcef6412020-10-07 13:09:054595 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 12:31:234596 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 09:28:414597 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 20:19:534598 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234599 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4600 WaitForEncodedFrame(CurrentTimeMs());
4601 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 20:19:534602 }
Per Kjellanderdcef6412020-10-07 13:09:054603 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 20:19:534604
mflodmancc3d4422017-08-03 15:27:514605 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 14:24:024606}
4607
Per Kjellanderf86cf4c2020-12-30 14:27:354608TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 15:53:224609 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034610 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:224611 kVideoLayersAllocation);
4612
4613 const int kDefaultFps = 30;
4614
4615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114616 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:224617
4618 video_source_.IncomingCapturedFrame(
4619 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4620 WaitForEncodedFrame(CurrentTimeMs());
4621 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4622 VideoLayersAllocation last_layer_allocation =
4623 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 08:07:114624 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 15:53:224625 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4626
4627 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 15:44:424628 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 15:53:224629 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 08:07:114630 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 15:53:224631 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4632
Erik Språng9d69cbe2020-10-22 15:44:424633 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 15:53:224634 int number_of_layers_allocation = 1;
4635 const int64_t start_time_ms = CurrentTimeMs();
4636 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4637 video_source_.IncomingCapturedFrame(
4638 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4639 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 15:53:224640 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4641 number_of_layers_allocation = sink_.number_of_layers_allocations();
4642 VideoLayersAllocation new_allocation =
4643 sink_.GetLastVideoLayersAllocation();
4644 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4645 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4646 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4647 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4648 .target_bitrate_per_temporal_layer,
4649 last_layer_allocation.active_spatial_layers[0]
4650 .target_bitrate_per_temporal_layer);
4651 last_layer_allocation = new_allocation;
4652 }
4653 }
4654 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4655 video_stream_encoder_->Stop();
4656}
4657
4658TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134659 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354660 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4661 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4662 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354663 VideoEncoderConfig video_encoder_config;
4664 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4665 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114666 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354667 video_encoder_config.content_type =
4668 VideoEncoderConfig::ContentType::kRealtimeVideo;
4669 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434670 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354671 VideoEncoder::GetDefaultVp8Settings());
4672 for (auto& layer : video_encoder_config.simulcast_layers) {
4673 layer.num_temporal_layers = 2;
4674 }
4675 // Simulcast layers are used for enabling/disabling streams.
4676 video_encoder_config.simulcast_layers[0].active = true;
4677 video_encoder_config.simulcast_layers[1].active = false;
4678 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034679 ConfigureEncoder(std::move(video_encoder_config),
4680 VideoStreamEncoder::BitrateAllocationCallbackType::
4681 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354682
4683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114684 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354685
4686 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4687 WaitForEncodedFrame(CurrentTimeMs());
4688 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4689 VideoLayersAllocation last_layer_allocation =
4690 sink_.GetLastVideoLayersAllocation();
4691
4692 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4693 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4694 .target_bitrate_per_temporal_layer,
4695 SizeIs(2));
4696 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4697 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4698 video_stream_encoder_->Stop();
4699}
4700
4701TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134702 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354703 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4704 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4705 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354706 VideoEncoderConfig video_encoder_config;
4707 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4708 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114709 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354710 video_encoder_config.content_type =
4711 VideoEncoderConfig::ContentType::kRealtimeVideo;
4712 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434713 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354714 VideoEncoder::GetDefaultVp8Settings());
4715 for (auto& layer : video_encoder_config.simulcast_layers) {
4716 layer.num_temporal_layers = 2;
4717 }
4718 // Simulcast layers are used for enabling/disabling streams.
4719 video_encoder_config.simulcast_layers[0].active = true;
4720 video_encoder_config.simulcast_layers[1].active = false;
4721 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034722 ConfigureEncoder(std::move(video_encoder_config),
4723 VideoStreamEncoder::BitrateAllocationCallbackType::
4724 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354725
4726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114727 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354728
4729 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4730 WaitForEncodedFrame(CurrentTimeMs());
4731 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4732 VideoLayersAllocation last_layer_allocation =
4733 sink_.GetLastVideoLayersAllocation();
4734
4735 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4736 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4737 .target_bitrate_per_temporal_layer,
4738 SizeIs(2));
4739 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4740
4741 video_stream_encoder_->Stop();
4742}
4743
4744TEST_F(VideoStreamEncoderTest,
4745 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4746 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4747 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354748 VideoEncoderConfig video_encoder_config;
4749 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4750 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114751 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354752 video_encoder_config.content_type =
4753 VideoEncoderConfig::ContentType::kRealtimeVideo;
4754 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4755 vp9_settings.numberOfSpatialLayers = 2;
4756 vp9_settings.numberOfTemporalLayers = 2;
4757 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4758 vp9_settings.automaticResizeOn = false;
4759 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434760 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354761 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034762 ConfigureEncoder(std::move(video_encoder_config),
4763 VideoStreamEncoder::BitrateAllocationCallbackType::
4764 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354765
4766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354768
4769 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4770 WaitForEncodedFrame(CurrentTimeMs());
4771 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4772 VideoLayersAllocation last_layer_allocation =
4773 sink_.GetLastVideoLayersAllocation();
4774
4775 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4776 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4777 .target_bitrate_per_temporal_layer,
4778 SizeIs(2));
4779 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4780 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4781 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4782 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4783 .target_bitrate_per_temporal_layer,
4784 SizeIs(2));
4785 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4786 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4787 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4788
4789 // Since full SVC is used, expect the top layer to utilize the full target
4790 // rate.
4791 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4792 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114793 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354794 video_stream_encoder_->Stop();
4795}
4796
4797TEST_F(VideoStreamEncoderTest,
4798 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4799 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4800 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 14:27:354801 VideoEncoderConfig video_encoder_config;
4802 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4803 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114804 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354805 video_encoder_config.content_type =
4806 VideoEncoderConfig::ContentType::kRealtimeVideo;
4807 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4808 vp9_settings.numberOfSpatialLayers = 2;
4809 vp9_settings.numberOfTemporalLayers = 2;
4810 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4811 vp9_settings.automaticResizeOn = false;
4812 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434813 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354814 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034815 ConfigureEncoder(std::move(video_encoder_config),
4816 VideoStreamEncoder::BitrateAllocationCallbackType::
4817 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354818
4819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114820 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354821
4822 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4823 WaitForEncodedFrame(CurrentTimeMs());
4824 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4825 VideoLayersAllocation last_layer_allocation =
4826 sink_.GetLastVideoLayersAllocation();
4827
4828 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4829 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4830 .target_bitrate_per_temporal_layer,
4831 SizeIs(1));
4832 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4833 .target_bitrate_per_temporal_layer,
4834 SizeIs(1));
4835 // Since full SVC is used, expect the top layer to utilize the full target
4836 // rate.
4837 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4838 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:114839 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354840 video_stream_encoder_->Stop();
4841}
4842
4843TEST_F(VideoStreamEncoderTest,
4844 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4845 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4846 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354847 VideoEncoderConfig video_encoder_config;
4848 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4849 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114850 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354851 video_encoder_config.content_type =
4852 VideoEncoderConfig::ContentType::kRealtimeVideo;
4853 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4854 vp9_settings.numberOfSpatialLayers = 2;
4855 vp9_settings.numberOfTemporalLayers = 2;
4856 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4857 vp9_settings.automaticResizeOn = false;
4858 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434859 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354860 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034861 ConfigureEncoder(std::move(video_encoder_config),
4862 VideoStreamEncoder::BitrateAllocationCallbackType::
4863 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354864
4865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114866 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354867
4868 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4869 WaitForEncodedFrame(CurrentTimeMs());
4870 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4871 VideoLayersAllocation last_layer_allocation =
4872 sink_.GetLastVideoLayersAllocation();
4873
4874 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4875 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4876 .target_bitrate_per_temporal_layer,
4877 SizeIs(2));
4878 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4879 .target_bitrate_per_temporal_layer,
4880 SizeIs(2));
4881 // Since KSVC is, spatial layers are independend except on key frames.
4882 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4883 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114884 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354885 video_stream_encoder_->Stop();
4886}
4887
4888TEST_F(VideoStreamEncoderTest,
4889 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4890 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4891 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4892 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354893 VideoEncoderConfig video_encoder_config;
4894 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4895 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114896 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354897 video_encoder_config.content_type =
4898 VideoEncoderConfig::ContentType::kRealtimeVideo;
4899 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4900 vp9_settings.numberOfSpatialLayers = 3;
4901 vp9_settings.numberOfTemporalLayers = 2;
4902 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4903 vp9_settings.automaticResizeOn = false;
4904 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434905 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354906 vp9_settings);
4907 // Simulcast layers are used for enabling/disabling streams.
4908 video_encoder_config.simulcast_layers.resize(3);
4909 video_encoder_config.simulcast_layers[0].active = false;
4910 video_encoder_config.simulcast_layers[1].active = true;
4911 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034912 ConfigureEncoder(std::move(video_encoder_config),
4913 VideoStreamEncoder::BitrateAllocationCallbackType::
4914 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354915
4916 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114917 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354918
4919 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4920 WaitForEncodedFrame(CurrentTimeMs());
4921 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4922 VideoLayersAllocation last_layer_allocation =
4923 sink_.GetLastVideoLayersAllocation();
4924
4925 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4926 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4927 .target_bitrate_per_temporal_layer,
4928 SizeIs(2));
4929 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4930 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4931
4932 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4933 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4934 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4935 .target_bitrate_per_temporal_layer,
4936 SizeIs(2));
4937 // Since full SVC is used, expect the top layer to utilize the full target
4938 // rate.
4939 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4940 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114941 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354942 video_stream_encoder_->Stop();
4943}
4944
4945TEST_F(VideoStreamEncoderTest,
4946 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4947 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4948 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4949 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354950 VideoEncoderConfig video_encoder_config;
4951 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4952 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114953 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354954 video_encoder_config.content_type =
4955 VideoEncoderConfig::ContentType::kRealtimeVideo;
4956 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4957 vp9_settings.numberOfSpatialLayers = 3;
4958 vp9_settings.numberOfTemporalLayers = 2;
4959 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4960 vp9_settings.automaticResizeOn = false;
4961 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434962 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354963 vp9_settings);
4964 // Simulcast layers are used for enabling/disabling streams.
4965 video_encoder_config.simulcast_layers.resize(3);
4966 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034967 ConfigureEncoder(std::move(video_encoder_config),
4968 VideoStreamEncoder::BitrateAllocationCallbackType::
4969 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354970
4971 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114972 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354973
4974 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4975 WaitForEncodedFrame(CurrentTimeMs());
4976 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4977 VideoLayersAllocation last_layer_allocation =
4978 sink_.GetLastVideoLayersAllocation();
4979
4980 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4981 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4982 .target_bitrate_per_temporal_layer,
4983 SizeIs(2));
4984 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4985 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4986
4987 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4988 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4989 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4990 .target_bitrate_per_temporal_layer,
4991 SizeIs(2));
4992 video_stream_encoder_->Stop();
4993}
4994
4995TEST_F(VideoStreamEncoderTest,
4996 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4997 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4998 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4999 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:355000 VideoEncoderConfig video_encoder_config;
5001 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5002 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:115003 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:355004 video_encoder_config.content_type =
5005 VideoEncoderConfig::ContentType::kRealtimeVideo;
5006 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5007 vp9_settings.numberOfSpatialLayers = 3;
5008 vp9_settings.numberOfTemporalLayers = 2;
5009 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5010 vp9_settings.automaticResizeOn = false;
5011 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435012 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:355013 vp9_settings);
5014 // Simulcast layers are used for enabling/disabling streams.
5015 video_encoder_config.simulcast_layers.resize(3);
5016 video_encoder_config.simulcast_layers[0].active = false;
5017 video_encoder_config.simulcast_layers[1].active = false;
5018 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:035019 ConfigureEncoder(std::move(video_encoder_config),
5020 VideoStreamEncoder::BitrateAllocationCallbackType::
5021 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:355022
5023 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115024 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:355025
5026 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5027 WaitForEncodedFrame(CurrentTimeMs());
5028 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5029 VideoLayersAllocation last_layer_allocation =
5030 sink_.GetLastVideoLayersAllocation();
5031
5032 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5033 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5034 .target_bitrate_per_temporal_layer,
5035 SizeIs(2));
5036 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5037 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5038 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5039 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:115040 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:355041 video_stream_encoder_->Stop();
5042}
5043
5044TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5045 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:035046 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 14:27:355047 kVideoLayersAllocation);
5048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115049 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:355050
5051 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5052 WaitForEncodedFrame(CurrentTimeMs());
5053 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5054 VideoLayersAllocation last_layer_allocation =
5055 sink_.GetLastVideoLayersAllocation();
5056
5057 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5058 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5059 .target_bitrate_per_temporal_layer,
5060 SizeIs(1));
5061 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5062 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:115063 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:355064 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5065 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5066 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5067 video_stream_encoder_->Stop();
5068}
5069
5070TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 15:53:225071 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5072 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035073 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:225074 kVideoLayersAllocation);
5075
5076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115077 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:225078
5079 video_source_.IncomingCapturedFrame(
5080 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5081 WaitForEncodedFrame(CurrentTimeMs());
5082 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5083 VideoLayersAllocation last_layer_allocation =
5084 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 08:07:115085 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 15:53:225086 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5087 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5088 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:115089 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 15:53:225090
5091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115092 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5093 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:225094 video_source_.IncomingCapturedFrame(
5095 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5096 WaitForEncodedFrame(CurrentTimeMs());
5097
5098 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5099 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5100 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5101 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5102 .target_bitrate_per_temporal_layer[0],
5103 DataRate::Zero());
5104
5105 video_stream_encoder_->Stop();
5106}
5107
Per Kjellander4190ce92020-12-15 16:24:555108TEST_F(VideoStreamEncoderTest,
5109 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5110 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035111 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 16:24:555112 kVideoLayersAllocation);
5113
5114 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115115 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5116 0, 0, 0);
Per Kjellander4190ce92020-12-15 16:24:555117
5118 video_source_.IncomingCapturedFrame(
5119 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5120 WaitForEncodedFrame(CurrentTimeMs());
5121 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5122 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5123 SizeIs(2));
5124 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5125 codec_width_);
5126 EXPECT_EQ(
5127 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5128 codec_height_);
5129
5130 video_source_.IncomingCapturedFrame(
5131 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5132 WaitForEncodedFrame(CurrentTimeMs());
5133 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5134 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5135 SizeIs(2));
5136 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5137 codec_width_ / 2);
5138 EXPECT_EQ(
5139 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5140 codec_height_ / 2);
5141
5142 video_stream_encoder_->Stop();
5143}
5144
Åsa Perssonc29cb2c2019-03-25 11:06:595145TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5146 // 2 TLs configured, temporal layers supported by encoder.
5147 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 13:09:055148 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035149 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055150 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595151 fake_encoder_.SetTemporalLayersSupported(0, true);
5152
5153 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 08:07:115154 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 11:06:595155 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:465156 kNumTemporalLayers, /*temporal_id*/ 0,
5157 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 08:07:115158 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 11:06:595159 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:465160 kNumTemporalLayers, /*temporal_id*/ 1,
5161 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595162 VideoBitrateAllocation expected_bitrate;
5163 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5164 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5165
5166 VerifyAllocatedBitrate(expected_bitrate);
5167 video_stream_encoder_->Stop();
5168}
5169
5170TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5171 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 13:09:055172 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035173 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055174 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595175 fake_encoder_.SetTemporalLayersSupported(0, false);
5176
5177 // Temporal layers not supported by the encoder.
5178 // Total bitrate should be at ti:0.
5179 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 08:07:115180 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 11:06:595181
5182 VerifyAllocatedBitrate(expected_bitrate);
5183 video_stream_encoder_->Stop();
5184}
5185
5186TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 13:09:055187 webrtc::test::ScopedFieldTrials field_trials(
5188 "WebRTC-Video-QualityScalerSettings/"
5189 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5190 // Reset encoder for field trials to take effect.
5191 ConfigureEncoder(video_encoder_config_.Copy());
5192
Åsa Perssonc29cb2c2019-03-25 11:06:595193 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 13:09:055194 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035195 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055196 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595197 fake_encoder_.SetTemporalLayersSupported(0, true);
5198 fake_encoder_.SetTemporalLayersSupported(1, false);
5199
5200 const int kS0Bps = 150000;
5201 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:465202 kS0Bps *
5203 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5204 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595205 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:465206 kS0Bps *
5207 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5208 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 08:07:115209 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 11:06:595210 // Temporal layers not supported by si:1.
5211 VideoBitrateAllocation expected_bitrate;
5212 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5213 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5214 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5215
5216 VerifyAllocatedBitrate(expected_bitrate);
5217 video_stream_encoder_->Stop();
5218}
5219
Niels Möller7dc26b72017-12-06 09:27:485220TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5221 const int kFrameWidth = 1280;
5222 const int kFrameHeight = 720;
5223 const int kFramerate = 24;
5224
Henrik Boström381d1092020-05-12 16:49:075225 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115226 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485227 test::FrameForwarder source;
5228 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415229 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485230
5231 // Insert a single frame, triggering initial configuration.
5232 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5233 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5234
5235 EXPECT_EQ(
5236 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5237 kDefaultFramerate);
5238
5239 // Trigger reconfigure encoder (without resetting the entire instance).
5240 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515241 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5242 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:115243 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 09:27:485244 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475245 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485246 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5247
5248 // Detector should be updated with fps limit from codec config.
5249 EXPECT_EQ(
5250 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5251 kFramerate);
5252
5253 // Trigger overuse, max framerate should be reduced.
5254 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5255 stats.input_frame_rate = kFramerate;
5256 stats_proxy_->SetMockStats(stats);
5257 video_stream_encoder_->TriggerCpuOveruse();
5258 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5259 int adapted_framerate =
5260 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5261 EXPECT_LT(adapted_framerate, kFramerate);
5262
5263 // Trigger underuse, max framerate should go back to codec configured fps.
5264 // Set extra low fps, to make sure it's actually reset, not just incremented.
5265 stats = stats_proxy_->GetStats();
5266 stats.input_frame_rate = adapted_framerate / 2;
5267 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335268 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485269 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5270 EXPECT_EQ(
5271 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5272 kFramerate);
5273
5274 video_stream_encoder_->Stop();
5275}
5276
5277TEST_F(VideoStreamEncoderTest,
5278 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5279 const int kFrameWidth = 1280;
5280 const int kFrameHeight = 720;
5281 const int kLowFramerate = 15;
5282 const int kHighFramerate = 25;
5283
Henrik Boström381d1092020-05-12 16:49:075284 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115285 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485286 test::FrameForwarder source;
5287 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415288 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485289
5290 // Trigger initial configuration.
5291 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515292 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5293 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 08:07:115294 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 09:27:485295 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 06:57:515296 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 07:51:475297 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485298 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5299
5300 EXPECT_EQ(
5301 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5302 kLowFramerate);
5303
5304 // Trigger overuse, max framerate should be reduced.
5305 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5306 stats.input_frame_rate = kLowFramerate;
5307 stats_proxy_->SetMockStats(stats);
5308 video_stream_encoder_->TriggerCpuOveruse();
5309 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5310 int adapted_framerate =
5311 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5312 EXPECT_LT(adapted_framerate, kLowFramerate);
5313
5314 // Reconfigure the encoder with a new (higher max framerate), max fps should
5315 // still respect the adaptation.
Åsa Persson17107062020-10-08 06:57:515316 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 09:27:485317 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5318 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475319 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485320 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5321
5322 EXPECT_EQ(
5323 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5324 adapted_framerate);
5325
5326 // Trigger underuse, max framerate should go back to codec configured fps.
5327 stats = stats_proxy_->GetStats();
5328 stats.input_frame_rate = adapted_framerate;
5329 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335330 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485331 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5332 EXPECT_EQ(
5333 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5334 kHighFramerate);
5335
5336 video_stream_encoder_->Stop();
5337}
5338
mflodmancc3d4422017-08-03 15:27:515339TEST_F(VideoStreamEncoderTest,
5340 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 11:21:075341 const int kFrameWidth = 1280;
5342 const int kFrameHeight = 720;
5343 const int kFramerate = 24;
5344
Henrik Boström381d1092020-05-12 16:49:075345 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115346 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 11:21:075347 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:515348 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415349 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 11:21:075350
5351 // Trigger initial configuration.
5352 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515353 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5354 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:115355 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 11:21:075356 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 15:27:515357 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475358 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:515359 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 11:21:075360
Niels Möller7dc26b72017-12-06 09:27:485361 EXPECT_EQ(
5362 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5363 kFramerate);
sprangfda496a2017-06-15 11:21:075364
5365 // Trigger overuse, max framerate should be reduced.
5366 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5367 stats.input_frame_rate = kFramerate;
5368 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 15:27:515369 video_stream_encoder_->TriggerCpuOveruse();
5370 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:485371 int adapted_framerate =
5372 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 11:21:075373 EXPECT_LT(adapted_framerate, kFramerate);
5374
5375 // Change degradation preference to not enable framerate scaling. Target
5376 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 16:49:075377 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:415378 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 09:27:485379 EXPECT_EQ(
5380 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5381 kFramerate);
sprangfda496a2017-06-15 11:21:075382
mflodmancc3d4422017-08-03 15:27:515383 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 11:21:075384}
5385
mflodmancc3d4422017-08-03 15:27:515386TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495387 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075389 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5390 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5391 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495392 const int kWidth = 640;
5393 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225394
asaperssonfab67072017-04-04 12:51:495395 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225396
5397 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525398 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225399
5400 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095401 EXPECT_TRUE_WAIT(
5402 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 15:02:225403
sprangc5d62e22017-04-03 06:53:045404 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 15:02:225405
asaperssonfab67072017-04-04 12:51:495406 // Next frame is scaled.
kthelgason2bc68642017-02-07 15:02:225407 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:495408 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 15:02:225409
5410 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525411 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225412
Henrik Boström2671dac2020-05-19 14:29:095413 EXPECT_TRUE_WAIT(
5414 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 15:02:225415
mflodmancc3d4422017-08-03 15:27:515416 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225417}
5418
mflodmancc3d4422017-08-03 15:27:515419TEST_F(VideoStreamEncoderTest,
5420 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495421 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075423 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5424 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5425 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495426 const int kWidth = 640;
5427 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225428
5429 // We expect the n initial frames to get dropped.
5430 int i;
5431 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 12:51:495432 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525433 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225434 }
5435 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 12:51:495436 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525437 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 15:02:225438
5439 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 12:51:495440 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:225441
mflodmancc3d4422017-08-03 15:27:515442 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225443}
5444
mflodmancc3d4422017-08-03 15:27:515445TEST_F(VideoStreamEncoderTest,
5446 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 12:51:495447 const int kWidth = 640;
5448 const int kHeight = 360;
Henrik Boström381d1092020-05-12 16:49:075449 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115450 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 15:02:225451
5452 // Set degradation preference.
mflodmancc3d4422017-08-03 15:27:515453 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415454 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 15:02:225455
asaperssonfab67072017-04-04 12:51:495456 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225457 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525458 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 15:02:225459
mflodmancc3d4422017-08-03 15:27:515460 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225461}
5462
mflodmancc3d4422017-08-03 15:27:515463TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 12:51:495464 const int kWidth = 640;
5465 const int kHeight = 360;
kthelgasonad9010c2017-02-14 08:46:515466 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:135467
5468 VideoEncoderConfig video_encoder_config;
5469 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5470 // Make format different, to force recreation of encoder.
5471 video_encoder_config.video_format.parameters["foo"] = "foo";
5472 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475473 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 16:49:075474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115475 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:185476
kthelgasonb83797b2017-02-14 19:57:255477 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 21:20:415478 video_stream_encoder_->SetSource(&video_source_,
5479 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 08:46:515480
asaperssonfab67072017-04-04 12:51:495481 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 08:46:515482 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525483 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 08:46:515484
mflodmancc3d4422017-08-03 15:27:515485 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 08:46:515486 fake_encoder_.SetQualityScaling(true);
5487}
5488
Åsa Persson139f4dc2019-08-02 07:29:585489TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5490 webrtc::test::ScopedFieldTrials field_trials(
5491 "WebRTC-Video-QualityScalerSettings/"
5492 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5493 // Reset encoder for field trials to take effect.
5494 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:115495 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5496 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 07:29:585497 const int kWidth = 640;
5498 const int kHeight = 360;
5499
Henrik Boström381d1092020-05-12 16:49:075500 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115501 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585502 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5503 // Frame should not be dropped.
5504 WaitForEncodedFrame(1);
5505
Henrik Boström381d1092020-05-12 16:49:075506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075507 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5508 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5509 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585510 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5511 // Frame should not be dropped.
5512 WaitForEncodedFrame(2);
5513
Henrik Boström381d1092020-05-12 16:49:075514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075515 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5516 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5517 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585518 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5519 // Expect to drop this frame, the wait should time out.
5520 ExpectDroppedFrame();
5521
5522 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095523 EXPECT_TRUE_WAIT(
5524 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 07:29:585525 video_stream_encoder_->Stop();
5526}
5527
Evan Shrubsolee3da1d32020-08-14 13:58:335528TEST_F(VideoStreamEncoderTest,
5529 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5530 webrtc::test::ScopedFieldTrials field_trials(
5531 "WebRTC-Video-QualityScalerSettings/"
5532 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5533 fake_encoder_.SetQualityScaling(false);
5534 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:115535 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5536 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 13:58:335537 const int kWidth = 640;
5538 const int kHeight = 360;
5539
5540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115541 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 13:58:335542 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5543 // Frame should not be dropped.
5544 WaitForEncodedFrame(1);
5545
5546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5547 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5548 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5549 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5550 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5551 // Frame should not be dropped.
5552 WaitForEncodedFrame(2);
5553
5554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5555 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5556 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5557 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5558 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5559 // Not dropped since quality scaling is disabled.
5560 WaitForEncodedFrame(3);
5561
5562 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 11:12:125563 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 13:58:335564 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5565
5566 video_stream_encoder_->Stop();
5567}
5568
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475569TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 08:07:115570 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475571 // Set simulcast.
5572 ResetEncoder("VP8", 3, 1, 1, false);
5573 fake_encoder_.SetQualityScaling(true);
5574 const int kWidth = 1280;
5575 const int kHeight = 720;
5576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115577 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475578 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5579 // Frame should not be dropped.
5580 WaitForEncodedFrame(1);
5581
5582 // Trigger QVGA "singlecast"
5583 // Update the config.
5584 VideoEncoderConfig video_encoder_config;
5585 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5586 &video_encoder_config);
Åsa Persson7f354f82021-02-04 14:52:155587 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:435588 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 14:52:155589 "VP8", /*max qp*/ 56, /*screencast*/ false,
5590 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475591 for (auto& layer : video_encoder_config.simulcast_layers) {
5592 layer.num_temporal_layers = 1;
5593 layer.max_framerate = kDefaultFramerate;
5594 }
Asa Persson606d3cb2021-10-04 08:07:115595 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475596 video_encoder_config.content_type =
5597 VideoEncoderConfig::ContentType::kRealtimeVideo;
5598
5599 video_encoder_config.simulcast_layers[0].active = true;
5600 video_encoder_config.simulcast_layers[1].active = false;
5601 video_encoder_config.simulcast_layers[2].active = false;
5602
5603 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5604 kMaxPayloadLength);
5605 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5606
5607 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5608 // Frame should not be dropped.
5609 WaitForEncodedFrame(2);
5610
5611 // Trigger HD "singlecast"
5612 video_encoder_config.simulcast_layers[0].active = false;
5613 video_encoder_config.simulcast_layers[1].active = false;
5614 video_encoder_config.simulcast_layers[2].active = true;
5615
5616 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5617 kMaxPayloadLength);
5618 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5619
5620 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5621 // Frame should be dropped because of initial frame drop.
5622 ExpectDroppedFrame();
5623
5624 // Expect the sink_wants to specify a scaled frame.
5625 EXPECT_TRUE_WAIT(
5626 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5627 video_stream_encoder_->Stop();
5628}
5629
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085630TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 08:07:115631 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085632 // Set simulcast.
5633 ResetEncoder("VP9", 1, 1, 3, false);
5634 fake_encoder_.SetQualityScaling(true);
5635 const int kWidth = 1280;
5636 const int kHeight = 720;
5637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115638 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085639 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5640 // Frame should not be dropped.
5641 WaitForEncodedFrame(1);
5642
5643 // Trigger QVGA "singlecast"
5644 // Update the config.
5645 VideoEncoderConfig video_encoder_config;
5646 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5647 &video_encoder_config);
5648 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5649 vp9_settings.numberOfSpatialLayers = 3;
5650 // Since only one layer is active - automatic resize should be enabled.
5651 vp9_settings.automaticResizeOn = true;
5652 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435653 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085654 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115655 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085656 video_encoder_config.content_type =
5657 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 14:22:115658 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085659 // which SVC layers are active.
5660 video_encoder_config.simulcast_layers.resize(3);
5661
5662 video_encoder_config.simulcast_layers[0].active = true;
5663 video_encoder_config.simulcast_layers[1].active = false;
5664 video_encoder_config.simulcast_layers[2].active = false;
5665
5666 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5667 kMaxPayloadLength);
5668 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5669
5670 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5671 // Frame should not be dropped.
5672 WaitForEncodedFrame(2);
5673
5674 // Trigger HD "singlecast"
5675 video_encoder_config.simulcast_layers[0].active = false;
5676 video_encoder_config.simulcast_layers[1].active = false;
5677 video_encoder_config.simulcast_layers[2].active = true;
5678
5679 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5680 kMaxPayloadLength);
5681 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5682
5683 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5684 // Frame should be dropped because of initial frame drop.
5685 ExpectDroppedFrame();
5686
5687 // Expect the sink_wants to specify a scaled frame.
5688 EXPECT_TRUE_WAIT(
5689 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5690 video_stream_encoder_->Stop();
5691}
5692
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475693TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055694 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5695 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5696 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5697 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5698 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5699 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5700 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5701 fake_encoder_.SetResolutionBitrateLimits(
5702 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5703
5704 VideoEncoderConfig video_encoder_config;
5705 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5706 &video_encoder_config);
5707 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5708 vp9_settings.numberOfSpatialLayers = 3;
5709 // Since only one layer is active - automatic resize should be enabled.
5710 vp9_settings.automaticResizeOn = true;
5711 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435712 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 08:20:055713 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115714 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 08:20:055715 video_encoder_config.content_type =
5716 VideoEncoderConfig::ContentType::kRealtimeVideo;
5717 // Simulcast layers are used to indicate which spatial layers are active.
5718 video_encoder_config.simulcast_layers.resize(3);
5719 video_encoder_config.simulcast_layers[0].active = false;
5720 video_encoder_config.simulcast_layers[1].active = true;
5721 video_encoder_config.simulcast_layers[2].active = false;
5722
5723 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5724 kMaxPayloadLength);
5725 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5726
5727 // The encoder bitrate limits for 360p should be used.
5728 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5729 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115730 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5731 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5732 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5733 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5734 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5735 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055736 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115737 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055738 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115739 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055740
5741 // The encoder bitrate limits for 270p should be used.
5742 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5743 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115744 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5745 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5746 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5747 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5748 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5749 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055750 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115751 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055752 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115753 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055754
5755 video_stream_encoder_->Stop();
5756}
5757
5758TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:515759 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5760 VideoEncoderConfig video_encoder_config;
5761 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5762 &video_encoder_config);
5763 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5764 vp9_settings.numberOfSpatialLayers = 3;
5765 // Since only one layer is active - automatic resize should be enabled.
5766 vp9_settings.automaticResizeOn = true;
5767 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435768 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 09:39:515769 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115770 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 09:39:515771 video_encoder_config.content_type =
5772 VideoEncoderConfig::ContentType::kRealtimeVideo;
5773 // Simulcast layers are used to indicate which spatial layers are active.
5774 video_encoder_config.simulcast_layers.resize(3);
5775 video_encoder_config.simulcast_layers[0].active = false;
5776 video_encoder_config.simulcast_layers[1].active = true;
5777 video_encoder_config.simulcast_layers[2].active = false;
5778
5779 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5780 kMaxPayloadLength);
5781 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5782
5783 // The default bitrate limits for 360p should be used.
5784 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195785 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5786 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515787 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5788 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115789 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5790 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5791 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5792 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5793 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5794 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515795 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115796 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515797 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115798 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515799
5800 // The default bitrate limits for 270p should be used.
5801 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 12:29:195802 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5803 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:515804 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5805 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115806 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5807 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5808 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5809 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5810 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5811 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515812 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115813 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515814 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115815 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515816
5817 video_stream_encoder_->Stop();
5818}
5819
5820TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5821 webrtc::test::ScopedFieldTrials field_trials(
5822 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5823 VideoEncoderConfig video_encoder_config;
5824 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5825 &video_encoder_config);
5826 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5827 vp9_settings.numberOfSpatialLayers = 3;
5828 // Since only one layer is active - automatic resize should be enabled.
5829 vp9_settings.automaticResizeOn = true;
5830 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435831 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 09:39:515832 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115833 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 09:39:515834 video_encoder_config.content_type =
5835 VideoEncoderConfig::ContentType::kRealtimeVideo;
5836 // Simulcast layers are used to indicate which spatial layers are active.
5837 video_encoder_config.simulcast_layers.resize(3);
5838 video_encoder_config.simulcast_layers[0].active = false;
5839 video_encoder_config.simulcast_layers[1].active = true;
5840 video_encoder_config.simulcast_layers[2].active = false;
5841
5842 // Reset encoder for field trials to take effect.
5843 ConfigureEncoder(video_encoder_config.Copy());
5844
5845 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5846 kMaxPayloadLength);
5847 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5848
5849 // The default bitrate limits for 360p should not be used.
5850 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195851 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5852 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515853 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5854 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115855 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5856 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5857 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5858 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5859 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5860 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515861 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115862 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515863
5864 video_stream_encoder_->Stop();
5865}
5866
5867TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5868 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5869 /*num_spatial_layers=*/1, /*screenshare=*/false);
5870
5871 // The default singlecast bitrate limits for 720p should not be used.
5872 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 12:29:195873 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5874 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 09:39:515875 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5876 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115877 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5878 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5879 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5880 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5881 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5882 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515883 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115884 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515885
5886 video_stream_encoder_->Stop();
5887}
5888
5889TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055890 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5891 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5892 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5893 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5894 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5895 fake_encoder_.SetResolutionBitrateLimits(
5896 {kEncoderLimits180p, kEncoderLimits720p});
5897
5898 VideoEncoderConfig video_encoder_config;
5899 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5900 &video_encoder_config);
5901 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5902 vp9_settings.numberOfSpatialLayers = 3;
5903 // Since only one layer is active - automatic resize should be enabled.
5904 vp9_settings.automaticResizeOn = true;
5905 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435906 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 08:20:055907 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115908 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 08:20:055909 video_encoder_config.content_type =
5910 VideoEncoderConfig::ContentType::kRealtimeVideo;
5911 // Simulcast layers are used to indicate which spatial layers are active.
5912 video_encoder_config.simulcast_layers.resize(3);
5913 video_encoder_config.simulcast_layers[0].active = true;
5914 video_encoder_config.simulcast_layers[1].active = false;
5915 video_encoder_config.simulcast_layers[2].active = false;
5916
5917 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5918 kMaxPayloadLength);
5919 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5920
5921 // Limits not applied on lowest stream, limits for 180p should not be used.
5922 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5923 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115924 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5925 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5926 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5927 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5928 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5929 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055930 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115931 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055932 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115933 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055934
5935 video_stream_encoder_->Stop();
5936}
5937
5938TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475939 InitialFrameDropActivatesWhenResolutionIncreases) {
5940 const int kWidth = 640;
5941 const int kHeight = 360;
5942
5943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115944 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475945 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5946 // Frame should not be dropped.
5947 WaitForEncodedFrame(1);
5948
5949 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115950 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475951 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5952 // Frame should not be dropped, bitrate not too low for frame.
5953 WaitForEncodedFrame(2);
5954
5955 // Incoming resolution increases.
5956 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5957 // Expect to drop this frame, bitrate too low for frame.
5958 ExpectDroppedFrame();
5959
5960 // Expect the sink_wants to specify a scaled frame.
5961 EXPECT_TRUE_WAIT(
5962 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5963 video_stream_encoder_->Stop();
5964}
5965
5966TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5967 const int kWidth = 640;
5968 const int kHeight = 360;
5969 // So that quality scaling doesn't happen by itself.
5970 fake_encoder_.SetQp(kQpHigh);
5971
5972 AdaptingFrameForwarder source(&time_controller_);
5973 source.set_adaptation_enabled(true);
5974 video_stream_encoder_->SetSource(
5975 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5976
5977 int timestamp = 1;
5978
5979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115980 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475981 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5982 WaitForEncodedFrame(timestamp);
5983 timestamp += 9000;
5984 // Long pause to disable all first BWE drop logic.
5985 AdvanceTime(TimeDelta::Millis(1000));
5986
5987 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115988 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475989 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5990 // Not dropped frame, as initial frame drop is disabled by now.
5991 WaitForEncodedFrame(timestamp);
5992 timestamp += 9000;
5993 AdvanceTime(TimeDelta::Millis(100));
5994
5995 // Quality adaptation down.
5996 video_stream_encoder_->TriggerQualityLow();
5997
5998 // Adaptation has an effect.
5999 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6000 5000);
6001
6002 // Frame isn't dropped as initial frame dropper is disabled.
6003 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6004 WaitForEncodedFrame(timestamp);
6005 timestamp += 9000;
6006 AdvanceTime(TimeDelta::Millis(100));
6007
6008 // Quality adaptation up.
6009 video_stream_encoder_->TriggerQualityHigh();
6010
6011 // Adaptation has an effect.
6012 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6013 5000);
6014
6015 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6016 // Frame should not be dropped, as initial framedropper is off.
6017 WaitForEncodedFrame(timestamp);
6018
6019 video_stream_encoder_->Stop();
6020}
6021
Åsa Persson7f354f82021-02-04 14:52:156022TEST_F(VideoStreamEncoderTest,
6023 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6024 const int kMinStartBps360p = 222000;
6025 fake_encoder_.SetResolutionBitrateLimits(
6026 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6027 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6028 800000)});
6029
6030 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6031 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6032 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6033 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6034 0, 0, 0);
6035 // Frame should not be dropped, bitrate not too low for frame.
6036 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6037 WaitForEncodedFrame(1);
6038
6039 // Incoming resolution increases, initial frame drop activates.
6040 // Frame should be dropped, link allocation too low for frame.
6041 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6042 ExpectDroppedFrame();
6043
6044 // Expect sink_wants to specify a scaled frame.
6045 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6046 5000);
6047 video_stream_encoder_->Stop();
6048}
6049
6050TEST_F(VideoStreamEncoderTest,
6051 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6052 const int kMinStartBps360p = 222000;
6053 fake_encoder_.SetResolutionBitrateLimits(
6054 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6055 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6056 800000)});
6057
6058 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6059 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6060 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6061 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6062 0, 0, 0);
6063 // Frame should not be dropped, bitrate not too low for frame.
6064 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6065 WaitForEncodedFrame(1);
6066
6067 // Incoming resolution increases, initial frame drop activates.
6068 // Frame should be dropped, link allocation not too low for frame.
6069 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6070 WaitForEncodedFrame(2);
6071
6072 video_stream_encoder_->Stop();
6073}
6074
Åsa Perssone644a032019-11-08 14:56:006075TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6076 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 13:28:486077 "WebRTC-Video-QualityRampupSettings/"
6078 "min_pixels:921600,min_duration_ms:2000/");
6079
6080 const int kWidth = 1280;
6081 const int kHeight = 720;
6082 const int kFps = 10;
6083 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 14:56:006084
6085 // Reset encoder for field trials to take effect.
6086 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 08:07:116087 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 09:34:326088 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 14:56:006089 ConfigureEncoder(std::move(config));
6090 fake_encoder_.SetQp(kQpLow);
6091
6092 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 12:31:236093 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 14:56:006094 source.set_adaptation_enabled(true);
6095 video_stream_encoder_->SetSource(&source,
6096 DegradationPreference::MAINTAIN_FRAMERATE);
6097
6098 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 08:07:116099 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 16:49:076100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116101 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:006102
6103 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 13:28:486104 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 14:56:006105 int64_t timestamp_ms = kFrameIntervalMs;
6106 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6107 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:096108 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6109 5000);
Åsa Perssone644a032019-11-08 14:56:006110
6111 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 16:49:076112 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6113 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:006114
Artem Titovab30d722021-07-27 14:22:116115 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 13:28:486116 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 14:56:006117 for (size_t i = 1; i <= 10; i++) {
6118 timestamp_ms += kFrameIntervalMs;
6119 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6120 WaitForEncodedFrame(timestamp_ms);
6121 }
Åsa Persson06defc42021-09-10 13:28:486122
6123 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6124 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6125 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6126 timestamp_ms += kFrameIntervalMs;
6127 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6128 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 14:56:006129 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6130 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6131
Åsa Persson06defc42021-09-10 13:28:486132 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 14:56:006133 timestamp_ms += kFrameIntervalMs;
6134 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6135 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 16:49:076136 // The ramp-up code involves the adaptation queue, give it time to execute.
6137 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 11:12:126138 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 12:08:396139 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 14:56:006140
6141 // Frame should not be adapted.
6142 timestamp_ms += kFrameIntervalMs;
6143 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6144 WaitForEncodedFrame(kWidth, kHeight);
6145 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6146
6147 video_stream_encoder_->Stop();
6148}
6149
mflodmancc3d4422017-08-03 15:27:516150TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 11:12:286151 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 16:19:316152 webrtc::test::ScopedFieldTrials field_trials(
6153 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 12:31:236154 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 11:12:286155 source.set_adaptation_enabled(true);
6156 video_stream_encoder_->SetSource(&source,
6157 DegradationPreference::MAINTAIN_FRAMERATE);
6158 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116159 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 11:12:286160 fake_encoder_.SetQp(kQpHigh + 1);
6161 const int kWidth = 1280;
6162 const int kHeight = 720;
6163 const int64_t kFrameIntervalMs = 100;
6164 int64_t timestamp_ms = kFrameIntervalMs;
6165 for (size_t i = 1; i <= 100; i++) {
6166 timestamp_ms += kFrameIntervalMs;
6167 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6168 WaitForEncodedFrame(timestamp_ms);
6169 }
6170 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6171 // for the first time.
6172 // TODO(eshr): We should avoid these waits by using threads with simulated
6173 // time.
6174 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6175 2000 * 2.5 * 2);
6176 timestamp_ms += kFrameIntervalMs;
6177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6178 WaitForEncodedFrame(timestamp_ms);
6179 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6180 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6181 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6182
6183 // Disable Quality scaling by turning off scaler on the encoder and
6184 // reconfiguring.
6185 fake_encoder_.SetQualityScaling(false);
6186 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6187 kMaxPayloadLength);
6188 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 09:11:556189 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 11:12:286190 // Since we turned off the quality scaler, the adaptations made by it are
6191 // removed.
6192 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6193 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6194
6195 video_stream_encoder_->Stop();
6196}
6197
6198TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:316199 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6200 const int kTooSmallWidth = 10;
6201 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 16:49:076202 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116203 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:316204
Taylor Brandstetter49fcc102018-05-16 21:20:416205 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:316206 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:516207 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416208 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 12:08:396209 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 08:47:316210 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6211
6212 // Trigger adapt down, too small frame, expect no change.
6213 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526214 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516215 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:396216 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:316217 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6218 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6219
mflodmancc3d4422017-08-03 15:27:516220 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:316221}
6222
mflodmancc3d4422017-08-03 15:27:516223TEST_F(VideoStreamEncoderTest,
6224 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226225 const int kTooSmallWidth = 10;
6226 const int kTooSmallHeight = 10;
6227 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 16:49:076228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116229 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226230
Taylor Brandstetter49fcc102018-05-16 21:20:416231 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:226232 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:416233 video_stream_encoder_->SetSource(&source,
6234 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:396235 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:226236 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6237 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6238
6239 // Trigger adapt down, expect limited framerate.
6240 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526241 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516242 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396243 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6245 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6246 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6247
6248 // Trigger adapt down, too small frame, expect no change.
6249 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526250 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:516251 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396252 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226253 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6254 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6255 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6256
mflodmancc3d4422017-08-03 15:27:516257 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226258}
6259
mflodmancc3d4422017-08-03 15:27:516260TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 08:12:526261 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 16:49:076262 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116263 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 07:51:476264 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 08:12:526265 const int kFrameWidth = 1280;
6266 const int kFrameHeight = 720;
6267 video_source_.IncomingCapturedFrame(
6268 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526269 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 15:27:516270 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:526271}
6272
sprangb1ca0732017-02-01 16:38:126273// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 15:27:516274TEST_F(VideoStreamEncoderTest,
6275 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 16:49:076276 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116277 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 16:38:126278
6279 const int kFrameWidth = 1280;
6280 const int kFrameHeight = 720;
6281 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 15:27:516282 // requested by
6283 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 16:38:126284 video_source_.set_adaptation_enabled(true);
6285
6286 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196287 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526288 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126289
6290 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516291 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 16:38:126292 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196293 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526294 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 16:38:126295
asaperssonfab67072017-04-04 12:51:496296 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 10:24:336297 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 16:38:126298 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196299 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526300 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126301
mflodmancc3d4422017-08-03 15:27:516302 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 16:38:126303}
sprangfe627f32017-03-29 15:24:596304
mflodmancc3d4422017-08-03 15:27:516305TEST_F(VideoStreamEncoderTest,
6306 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-03 06:53:046307 const int kFrameWidth = 1280;
6308 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:046309
Henrik Boström381d1092020-05-12 16:49:076310 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116311 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516312 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416313 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046314 video_source_.set_adaptation_enabled(true);
6315
Tomas Gunnarsson612445e2020-09-21 12:31:236316 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046317
6318 video_source_.IncomingCapturedFrame(
6319 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526320 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046321
6322 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 15:27:516323 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046324
6325 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 14:06:526326 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046327 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046328 video_source_.IncomingCapturedFrame(
6329 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526330 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046331 }
6332
6333 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 15:27:516334 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046335 int num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526336 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046337 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046338 video_source_.IncomingCapturedFrame(
6339 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526340 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046341 ++num_frames_dropped;
6342 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566343 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046344 }
6345 }
6346
sprang4847ae62017-06-27 14:06:526347 // Add some slack to account for frames dropped by the frame dropper.
6348 const int kErrorMargin = 1;
6349 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046350 kErrorMargin);
6351
6352 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 15:27:516353 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046354 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 08:42:196355 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046356 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046357 video_source_.IncomingCapturedFrame(
6358 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526359 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046360 ++num_frames_dropped;
6361 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566362 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046363 }
6364 }
sprang4847ae62017-06-27 14:06:526365 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-03 06:53:046366 kErrorMargin);
6367
6368 // Go back up one step.
Henrik Boström91aa7322020-04-28 10:24:336369 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046370 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526371 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046372 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046373 video_source_.IncomingCapturedFrame(
6374 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526375 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046376 ++num_frames_dropped;
6377 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566378 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046379 }
6380 }
sprang4847ae62017-06-27 14:06:526381 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046382 kErrorMargin);
6383
6384 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 10:24:336385 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046386 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526387 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046388 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046389 video_source_.IncomingCapturedFrame(
6390 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526391 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046392 ++num_frames_dropped;
6393 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566394 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046395 }
6396 }
6397 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6398
mflodmancc3d4422017-08-03 15:27:516399 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046400}
6401
mflodmancc3d4422017-08-03 15:27:516402TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-03 06:53:046403 const int kFramerateFps = 5;
6404 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-03 06:53:046405 const int kFrameWidth = 1280;
6406 const int kFrameHeight = 720;
6407
sprang4847ae62017-06-27 14:06:526408 // Reconfigure encoder with two temporal layers and screensharing, which will
6409 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 07:51:476410 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 14:06:526411
Henrik Boström381d1092020-05-12 16:49:076412 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116413 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516414 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416415 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046416 video_source_.set_adaptation_enabled(true);
6417
Tomas Gunnarsson612445e2020-09-21 12:31:236418 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046419
6420 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-09 01:04:296421 rtc::VideoSinkWants last_wants;
6422 do {
6423 last_wants = video_source_.sink_wants();
6424
sprangc5d62e22017-04-03 06:53:046425 // Insert frames to get a new fps estimate...
6426 for (int j = 0; j < kFramerateFps; ++j) {
6427 video_source_.IncomingCapturedFrame(
6428 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-09 01:04:296429 if (video_source_.last_sent_width()) {
6430 sink_.WaitForEncodedFrame(timestamp_ms);
6431 }
sprangc5d62e22017-04-03 06:53:046432 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:236433 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-03 06:53:046434 }
6435 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 15:27:516436 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:296437 } while (video_source_.sink_wants().max_framerate_fps <
6438 last_wants.max_framerate_fps);
sprangc5d62e22017-04-03 06:53:046439
Evan Shrubsole5cd7eb82020-05-25 12:08:396440 EXPECT_THAT(video_source_.sink_wants(),
6441 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-14 06:25:226442
mflodmancc3d4422017-08-03 15:27:516443 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046444}
asaperssonf7e294d2017-06-14 06:25:226445
mflodmancc3d4422017-08-03 15:27:516446TEST_F(VideoStreamEncoderTest,
6447 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226448 const int kWidth = 1280;
6449 const int kHeight = 720;
6450 const int64_t kFrameIntervalMs = 150;
6451 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116453 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226454
Taylor Brandstetter49fcc102018-05-16 21:20:416455 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236456 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226457 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416458 video_stream_encoder_->SetSource(&source,
6459 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226460 timestamp_ms += kFrameIntervalMs;
6461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526462 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396463 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226464 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6466 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6467
6468 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516469 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226470 timestamp_ms += kFrameIntervalMs;
6471 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526472 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396473 EXPECT_THAT(source.sink_wants(),
6474 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226475 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6476 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6477 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6478
6479 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516480 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226481 timestamp_ms += kFrameIntervalMs;
6482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526483 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546484 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226485 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6487 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6488
6489 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516490 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226491 timestamp_ms += kFrameIntervalMs;
6492 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526493 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546494 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6496 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6497 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6498
6499 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516500 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226501 timestamp_ms += kFrameIntervalMs;
6502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526503 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546504 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226505 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6506 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6507 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6508
6509 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516510 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226511 timestamp_ms += kFrameIntervalMs;
6512 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526513 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546514 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6517 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6518
6519 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516520 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226521 timestamp_ms += kFrameIntervalMs;
6522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526523 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546524 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6526 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6527 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6528
6529 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 15:27:516530 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226531 timestamp_ms += kFrameIntervalMs;
6532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526533 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546534 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226535 rtc::VideoSinkWants last_wants = source.sink_wants();
6536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6538 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6539
6540 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 15:27:516541 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226542 timestamp_ms += kFrameIntervalMs;
6543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526544 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546545 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-14 06:25:226546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6548 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6549
Åsa Perssonf5f7e8e2021-06-09 20:55:386550 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516551 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226552 timestamp_ms += kFrameIntervalMs;
6553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526554 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546555 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6558 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
6560 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516561 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546565 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6569
6570 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516571 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226572 timestamp_ms += kFrameIntervalMs;
6573 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526574 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546575 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6578 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6579
6580 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516581 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226582 timestamp_ms += kFrameIntervalMs;
6583 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526584 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546585 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6588 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6589
6590 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516591 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226592 timestamp_ms += kFrameIntervalMs;
6593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526594 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546595 EXPECT_THAT(source.sink_wants(), FpsMax());
6596 EXPECT_EQ(source.sink_wants().max_pixel_count,
6597 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-14 06:25:226598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6599 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6600 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6601
6602 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516603 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226604 timestamp_ms += kFrameIntervalMs;
6605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526606 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546607 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6610 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6611
Åsa Persson30ab0152019-08-27 10:22:336612 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516613 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226614 timestamp_ms += kFrameIntervalMs;
6615 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526616 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546617 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396618 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226619 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6620 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6621 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6622
6623 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516624 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396625 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226626 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6627
mflodmancc3d4422017-08-03 15:27:516628 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226629}
6630
mflodmancc3d4422017-08-03 15:27:516631TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-14 06:25:226632 const int kWidth = 1280;
6633 const int kHeight = 720;
6634 const int64_t kFrameIntervalMs = 150;
6635 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116637 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226638
Taylor Brandstetter49fcc102018-05-16 21:20:416639 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236640 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226641 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416642 video_stream_encoder_->SetSource(&source,
6643 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226644 timestamp_ms += kFrameIntervalMs;
6645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526646 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396647 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6649 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6650 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6651 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6652 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6654
6655 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516656 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226657 timestamp_ms += kFrameIntervalMs;
6658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526659 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396660 EXPECT_THAT(source.sink_wants(),
6661 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226662 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6663 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6664 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6665 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6666 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6667 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6668
6669 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516670 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226671 timestamp_ms += kFrameIntervalMs;
6672 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526673 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546674 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226675 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6677 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6678 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6679 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6680 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6681
6682 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516683 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226684 timestamp_ms += kFrameIntervalMs;
6685 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526686 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546687 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 08:45:296688 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-14 06:25:226689 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6690 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6691 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6692 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6693 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6694
Evan Shrubsole64469032020-06-11 08:45:296695 // Trigger cpu adapt up, expect no change since QP is most limited.
6696 {
6697 // Store current sink wants since we expect no change and if there is no
6698 // change then last_wants() is not updated.
6699 auto previous_sink_wants = source.sink_wants();
6700 video_stream_encoder_->TriggerCpuUnderuse();
6701 timestamp_ms += kFrameIntervalMs;
6702 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6703 WaitForEncodedFrame(timestamp_ms);
6704 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6705 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6706 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6707 }
6708
6709 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6710 video_stream_encoder_->TriggerQualityHigh();
6711 timestamp_ms += kFrameIntervalMs;
6712 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6713 WaitForEncodedFrame(timestamp_ms);
6714 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6715 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6716 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6717 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6719 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6720 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6721
6722 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6723 // expect increased resolution (960x540@30fps).
6724 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336725 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226726 timestamp_ms += kFrameIntervalMs;
6727 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526728 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 08:45:296729 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226730 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6731 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6732 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6733 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6734 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296735 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226736
Evan Shrubsole64469032020-06-11 08:45:296737 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6738 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516739 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336740 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226741 timestamp_ms += kFrameIntervalMs;
6742 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526743 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546744 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396745 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226746 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6747 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6748 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6749 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6750 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296751 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226752
6753 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516754 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396755 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226756 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296757 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226758
mflodmancc3d4422017-08-03 15:27:516759 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226760}
6761
mflodmancc3d4422017-08-03 15:27:516762TEST_F(VideoStreamEncoderTest,
6763 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-14 06:25:226764 const int kWidth = 640;
6765 const int kHeight = 360;
6766 const int kFpsLimit = 15;
6767 const int64_t kFrameIntervalMs = 150;
6768 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116770 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226771
Taylor Brandstetter49fcc102018-05-16 21:20:416772 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236773 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226774 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416775 video_stream_encoder_->SetSource(&source,
6776 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226777 timestamp_ms += kFrameIntervalMs;
6778 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526779 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396780 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226781 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6782 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6785 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6786 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6787
6788 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516789 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226790 timestamp_ms += kFrameIntervalMs;
6791 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526792 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396793 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226794 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6795 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6796 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6797 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6798 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6799 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6800
6801 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516802 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226803 timestamp_ms += kFrameIntervalMs;
6804 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526805 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546806 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226807 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296808 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:226809 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6810 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6811 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6812 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6813
Evan Shrubsole64469032020-06-11 08:45:296814 // Trigger cpu adapt up, expect no change because quality is most limited.
6815 {
6816 auto previous_sink_wants = source.sink_wants();
6817 // Store current sink wants since we expect no change ind if there is no
6818 // change then last__wants() is not updated.
6819 video_stream_encoder_->TriggerCpuUnderuse();
6820 timestamp_ms += kFrameIntervalMs;
6821 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6822 WaitForEncodedFrame(timestamp_ms);
6823 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6824 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6825 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6826 }
6827
6828 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6829 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226830 timestamp_ms += kFrameIntervalMs;
6831 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526832 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546833 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226834 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6835 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6836 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296837 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6838 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6839 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226840
Evan Shrubsole64469032020-06-11 08:45:296841 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516842 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 08:45:296843 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226844 timestamp_ms += kFrameIntervalMs;
6845 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526846 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396847 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226848 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6850 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6851 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6852 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296853 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226854
6855 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516856 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396857 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226858 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296859 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226860
mflodmancc3d4422017-08-03 15:27:516861 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226862}
6863
mflodmancc3d4422017-08-03 15:27:516864TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 13:53:486865 const int kFrameWidth = 1920;
6866 const int kFrameHeight = 1080;
6867 // 3/4 of 1920.
6868 const int kAdaptedFrameWidth = 1440;
6869 // 3/4 of 1080 rounded down to multiple of 4.
6870 const int kAdaptedFrameHeight = 808;
6871 const int kFramerate = 24;
6872
Henrik Boström381d1092020-05-12 16:49:076873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116874 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 13:53:486875 // Trigger reconfigure encoder (without resetting the entire instance).
6876 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 10:57:586877 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6878 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:116879 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 13:53:486880 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:436881 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 15:27:516882 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:476883 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:516884 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 13:53:486885
6886 video_source_.set_adaptation_enabled(true);
6887
6888 video_source_.IncomingCapturedFrame(
6889 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526890 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 13:53:486891
6892 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516893 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 13:53:486894 video_source_.IncomingCapturedFrame(
6895 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526896 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 13:53:486897
mflodmancc3d4422017-08-03 15:27:516898 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 13:53:486899}
6900
mflodmancc3d4422017-08-03 15:27:516901TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 14:06:526902 const int kFrameWidth = 1280;
6903 const int kFrameHeight = 720;
6904 const int kLowFps = 2;
6905 const int kHighFps = 30;
6906
Henrik Boström381d1092020-05-12 16:49:076907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116908 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:526909
Tomas Gunnarsson612445e2020-09-21 12:31:236910 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:526911 max_framerate_ = kLowFps;
6912
6913 // Insert 2 seconds of 2fps video.
6914 for (int i = 0; i < kLowFps * 2; ++i) {
6915 video_source_.IncomingCapturedFrame(
6916 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6917 WaitForEncodedFrame(timestamp_ms);
6918 timestamp_ms += 1000 / kLowFps;
6919 }
6920
6921 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 16:49:076922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116923 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:526924 video_source_.IncomingCapturedFrame(
6925 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6926 WaitForEncodedFrame(timestamp_ms);
6927 timestamp_ms += 1000 / kLowFps;
6928
6929 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6930
6931 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 08:48:486932 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 14:06:526933 const int kFrameIntervalMs = 1000 / kHighFps;
6934 max_framerate_ = kHighFps;
6935 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6936 video_source_.IncomingCapturedFrame(
6937 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6938 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6939 // be dropped if the encoder hans't been updated with the new higher target
6940 // framerate yet, causing it to overshoot the target bitrate and then
6941 // suffering the wrath of the media optimizer.
6942 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6943 timestamp_ms += kFrameIntervalMs;
6944 }
6945
6946 // Don expect correct measurement just yet, but it should be higher than
6947 // before.
6948 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6949
mflodmancc3d4422017-08-03 15:27:516950 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:526951}
6952
mflodmancc3d4422017-08-03 15:27:516953TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 14:06:526954 const int kFrameWidth = 1280;
6955 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 13:09:056956 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:036957 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:056958 kVideoBitrateAllocation);
sprang4847ae62017-06-27 14:06:526959
Henrik Boström381d1092020-05-12 16:49:076960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116961 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516962 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 14:06:526963
6964 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 12:31:236965 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:526966 video_source_.IncomingCapturedFrame(
6967 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6968 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 13:09:056969 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:526970
6971 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 16:49:076972 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116973 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 14:06:526974
6975 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 08:48:486976 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:236977 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 14:06:526978
Per Kjellanderdcef6412020-10-07 13:09:056979 // No more allocations has been made.
sprang4847ae62017-06-27 14:06:526980 video_source_.IncomingCapturedFrame(
6981 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6982 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 13:09:056983 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:526984
mflodmancc3d4422017-08-03 15:27:516985 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:526986}
ilnik6b826ef2017-06-16 13:53:486987
Niels Möller4db138e2018-04-19 07:04:136988TEST_F(VideoStreamEncoderTest,
6989 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6990 const int kFrameWidth = 1280;
6991 const int kFrameHeight = 720;
6992 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 16:49:076993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:136995 video_source_.IncomingCapturedFrame(
6996 CreateFrame(1, kFrameWidth, kFrameHeight));
6997 WaitForEncodedFrame(1);
6998 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6999 .low_encode_usage_threshold_percent,
7000 default_options.low_encode_usage_threshold_percent);
7001 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7002 .high_encode_usage_threshold_percent,
7003 default_options.high_encode_usage_threshold_percent);
7004 video_stream_encoder_->Stop();
7005}
7006
7007TEST_F(VideoStreamEncoderTest,
7008 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7009 const int kFrameWidth = 1280;
7010 const int kFrameHeight = 720;
7011 CpuOveruseOptions hardware_options;
7012 hardware_options.low_encode_usage_threshold_percent = 150;
7013 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 11:42:187014 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 07:04:137015
Henrik Boström381d1092020-05-12 16:49:077016 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117017 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:137018 video_source_.IncomingCapturedFrame(
7019 CreateFrame(1, kFrameWidth, kFrameHeight));
7020 WaitForEncodedFrame(1);
7021 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7022 .low_encode_usage_threshold_percent,
7023 hardware_options.low_encode_usage_threshold_percent);
7024 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7025 .high_encode_usage_threshold_percent,
7026 hardware_options.high_encode_usage_threshold_percent);
7027 video_stream_encoder_->Stop();
7028}
7029
Jakob Ivarsson461b1d92021-01-22 15:27:437030TEST_F(VideoStreamEncoderTest,
7031 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7032 const int kFrameWidth = 1280;
7033 const int kFrameHeight = 720;
7034
7035 const CpuOveruseOptions default_options;
7036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117037 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 15:27:437038 video_source_.IncomingCapturedFrame(
7039 CreateFrame(1, kFrameWidth, kFrameHeight));
7040 WaitForEncodedFrame(1);
7041 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7042 .low_encode_usage_threshold_percent,
7043 default_options.low_encode_usage_threshold_percent);
7044 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7045 .high_encode_usage_threshold_percent,
7046 default_options.high_encode_usage_threshold_percent);
7047
7048 CpuOveruseOptions hardware_options;
7049 hardware_options.low_encode_usage_threshold_percent = 150;
7050 hardware_options.high_encode_usage_threshold_percent = 200;
7051 fake_encoder_.SetIsHardwareAccelerated(true);
7052
7053 video_source_.IncomingCapturedFrame(
7054 CreateFrame(2, kFrameWidth, kFrameHeight));
7055 WaitForEncodedFrame(2);
7056
7057 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7058 .low_encode_usage_threshold_percent,
7059 hardware_options.low_encode_usage_threshold_percent);
7060 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7061 .high_encode_usage_threshold_percent,
7062 hardware_options.high_encode_usage_threshold_percent);
7063
7064 video_stream_encoder_->Stop();
7065}
7066
Niels Möller6bb5ab92019-01-11 10:11:107067TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7068 const int kFrameWidth = 320;
7069 const int kFrameHeight = 240;
7070 const int kFps = 30;
Asa Persson606d3cb2021-10-04 08:07:117071 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 10:11:107072 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7073
Henrik Boström381d1092020-05-12 16:49:077074 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117075 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107076
Tomas Gunnarsson612445e2020-09-21 12:31:237077 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:107078 max_framerate_ = kFps;
7079
7080 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7081 fake_encoder_.SimulateOvershoot(1.0);
7082 int num_dropped = 0;
7083 for (int i = 0; i < kNumFramesInRun; ++i) {
7084 video_source_.IncomingCapturedFrame(
7085 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7086 // Wait up to two frame durations for a frame to arrive.
7087 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7088 ++num_dropped;
7089 }
7090 timestamp_ms += 1000 / kFps;
7091 }
7092
Erik Språnga8d48ab2019-02-08 13:17:407093 // Framerate should be measured to be near the expected target rate.
7094 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7095
7096 // Frame drops should be within 5% of expected 0%.
7097 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:107098
7099 // Make encoder produce frames at double the expected bitrate during 3 seconds
7100 // of video, verify number of drops. Rate needs to be slightly changed in
7101 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 15:20:177102 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 15:44:427103 const RateControlSettings trials =
7104 RateControlSettings::ParseFromFieldTrials();
7105 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 15:20:177106 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 15:44:427107 // frame dropping since the adjuter will try to just lower the target
7108 // bitrate rather than drop frames. If network headroom can be used, it
7109 // doesn't push back as hard so we don't need quite as much overshoot.
7110 // These numbers are unfortunately a bit magical but there's not trivial
7111 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 10:10:097112 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 15:20:177113 }
7114 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 16:49:077115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117116 kTargetBitrate + DataRate::KilobitsPerSec(1),
7117 kTargetBitrate + DataRate::KilobitsPerSec(1),
7118 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107119 num_dropped = 0;
7120 for (int i = 0; i < kNumFramesInRun; ++i) {
7121 video_source_.IncomingCapturedFrame(
7122 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7123 // Wait up to two frame durations for a frame to arrive.
7124 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7125 ++num_dropped;
7126 }
7127 timestamp_ms += 1000 / kFps;
7128 }
7129
Henrik Boström381d1092020-05-12 16:49:077130 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117131 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 13:17:407132
7133 // Target framerate should be still be near the expected target, despite
7134 // the frame drops.
7135 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7136
7137 // Frame drops should be within 5% of expected 50%.
7138 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:107139
7140 video_stream_encoder_->Stop();
7141}
7142
7143TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7144 const int kFrameWidth = 320;
7145 const int kFrameHeight = 240;
7146 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 08:07:117147 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 10:11:107148
7149 ASSERT_GT(max_framerate_, kActualInputFps);
7150
Tomas Gunnarsson612445e2020-09-21 12:31:237151 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:107152 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 16:49:077153 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117154 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107155
7156 // Insert 3 seconds of video, with an input fps lower than configured max.
7157 for (int i = 0; i < kActualInputFps * 3; ++i) {
7158 video_source_.IncomingCapturedFrame(
7159 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7160 // Wait up to two frame durations for a frame to arrive.
7161 WaitForEncodedFrame(timestamp_ms);
7162 timestamp_ms += 1000 / kActualInputFps;
7163 }
7164
7165 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7166
7167 video_stream_encoder_->Stop();
7168}
7169
Markus Handell9a478b52021-11-18 15:07:017170TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267171 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 15:07:017172 test::FrameForwarder source;
7173 video_stream_encoder_->SetSource(&source,
7174 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 16:49:077175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117176 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267177
Markus Handell9a478b52021-11-18 15:07:017178 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267179 WaitForEncodedFrame(1);
7180 // On the very first frame full update should be forced.
7181 rect = fake_encoder_.GetLastUpdateRect();
7182 EXPECT_EQ(rect.offset_x, 0);
7183 EXPECT_EQ(rect.offset_y, 0);
7184 EXPECT_EQ(rect.height, codec_height_);
7185 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 15:07:017186 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7187 // scheduled for processing during encoder queue processing of frame 2.
7188 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7189 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267190 WaitForEncodedFrame(3);
7191 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7192 rect = fake_encoder_.GetLastUpdateRect();
7193 EXPECT_EQ(rect.offset_x, 1);
7194 EXPECT_EQ(rect.offset_y, 0);
7195 EXPECT_EQ(rect.width, 10);
7196 EXPECT_EQ(rect.height, 1);
7197
Markus Handell9a478b52021-11-18 15:07:017198 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267199 WaitForEncodedFrame(4);
7200 // Previous frame was encoded, so no accumulation should happen.
7201 rect = fake_encoder_.GetLastUpdateRect();
7202 EXPECT_EQ(rect.offset_x, 0);
7203 EXPECT_EQ(rect.offset_y, 0);
7204 EXPECT_EQ(rect.width, 1);
7205 EXPECT_EQ(rect.height, 1);
7206
7207 video_stream_encoder_->Stop();
7208}
7209
Erik Språngd7329ca2019-02-21 20:19:537210TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 16:49:077211 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117212 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537213
7214 // First frame is always keyframe.
7215 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7216 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 14:43:587217 EXPECT_THAT(
7218 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127219 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537220
7221 // Insert delta frame.
7222 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7223 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 14:43:587224 EXPECT_THAT(
7225 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127226 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537227
7228 // Request next frame be a key-frame.
7229 video_stream_encoder_->SendKeyFrame();
7230 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7231 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 14:43:587232 EXPECT_THAT(
7233 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127234 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537235
7236 video_stream_encoder_->Stop();
7237}
7238
7239TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7240 // Setup simulcast with three streams.
7241 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117243 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7244 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537245 // Wait for all three layers before triggering event.
7246 sink_.SetNumExpectedLayers(3);
7247
7248 // First frame is always keyframe.
7249 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7250 WaitForEncodedFrame(1);
7251 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127252 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7253 VideoFrameType::kVideoFrameKey,
7254 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537255
7256 // Insert delta frame.
7257 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7258 WaitForEncodedFrame(2);
7259 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127260 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7261 VideoFrameType::kVideoFrameDelta,
7262 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537263
7264 // Request next frame be a key-frame.
7265 // Only first stream is configured to produce key-frame.
7266 video_stream_encoder_->SendKeyFrame();
7267 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7268 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 11:45:397269
7270 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7271 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 20:19:537272 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127273 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 11:45:397274 VideoFrameType::kVideoFrameKey,
7275 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537276
7277 video_stream_encoder_->Stop();
7278}
7279
Mirta Dvornicic28f0eb22019-05-28 14:30:167280TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237281 // SPS contains VUI with restrictions on the maximum number of reordered
7282 // pictures, there is no need to rewrite the bitstream to enable faster
7283 // decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167284 ResetEncoder("H264", 1, 1, 1, false);
7285
Mirta Dvornicic97910da2020-07-14 13:29:237286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117287 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 13:29:237288 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167289
Mirta Dvornicic97910da2020-07-14 13:29:237290 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 08:07:117291 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167292
Mirta Dvornicic97910da2020-07-14 13:29:237293 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7294 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167295
7296 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 08:07:117297 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167298
7299 video_stream_encoder_->Stop();
7300}
7301
7302TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237303 // SPS does not contain VUI, the bitstream is will be rewritten with added
7304 // VUI with restrictions on the maximum number of reordered pictures to
7305 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167306 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7307 0x00, 0x00, 0x03, 0x03, 0xF4,
7308 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 14:30:167309 ResetEncoder("H264", 1, 1, 1, false);
7310
Mirta Dvornicic97910da2020-07-14 13:29:237311 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117312 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 13:29:237313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167314
Mirta Dvornicic97910da2020-07-14 13:29:237315 fake_encoder_.SetEncodedImageData(
7316 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167317
Mirta Dvornicic97910da2020-07-14 13:29:237318 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7319 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167320
7321 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 08:07:117322 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167323
7324 video_stream_encoder_->Stop();
7325}
7326
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147327TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7328 const int kFrameWidth = 1280;
7329 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 08:07:117330 const DataRate kTargetBitrate =
7331 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147332
Henrik Boström381d1092020-05-12 16:49:077333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117334 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147335 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7336
7337 // Insert a first video frame. It should be dropped because of downscale in
7338 // resolution.
Tomas Gunnarsson612445e2020-09-21 12:31:237339 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147340 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7341 frame.set_rotation(kVideoRotation_270);
7342 video_source_.IncomingCapturedFrame(frame);
7343
7344 ExpectDroppedFrame();
7345
7346 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237347 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147348 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7349 frame.set_rotation(kVideoRotation_90);
7350 video_source_.IncomingCapturedFrame(frame);
7351
7352 WaitForEncodedFrame(timestamp_ms);
7353 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7354
7355 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237356 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147357 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7358 frame.set_rotation(kVideoRotation_180);
7359 video_source_.IncomingCapturedFrame(frame);
7360
7361 WaitForEncodedFrame(timestamp_ms);
7362 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7363
7364 video_stream_encoder_->Stop();
7365}
7366
Erik Språng5056af02019-09-02 13:53:117367TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7368 const int kFrameWidth = 320;
7369 const int kFrameHeight = 180;
7370
7371 // Initial rate.
Henrik Boström381d1092020-05-12 16:49:077372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077373 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7374 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7375 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 13:53:117376 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487377 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327378 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117379
7380 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237381 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 13:53:117382 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7383 frame.set_rotation(kVideoRotation_270);
7384 video_source_.IncomingCapturedFrame(frame);
7385 WaitForEncodedFrame(timestamp_ms);
7386
7387 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 08:07:117388 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 17:46:077389 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7390 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 16:49:077391 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 13:53:117392 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 11:36:557393 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 13:53:117394 /*link_allocation=*/target_rate,
7395 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487396 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327397 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117398 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7399
7400 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7401 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7402 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 17:46:077403 DataRate allocation_sum =
7404 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 13:53:117405 EXPECT_EQ(min_rate, allocation_sum);
7406 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7407
7408 video_stream_encoder_->Stop();
7409}
7410
Rasmus Brandt5cad55b2019-12-19 08:47:117411TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 16:49:077412 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117413 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 10:50:237414 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 12:31:237415 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 10:50:237416 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7417 WaitForEncodedFrame(1);
7418
7419 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7420 ASSERT_TRUE(prev_rate_settings.has_value());
7421 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7422 kDefaultFramerate);
7423
7424 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7425 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7426 timestamp_ms += 1000 / kDefaultFramerate;
7427 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7428 WaitForEncodedFrame(timestamp_ms);
7429 }
7430 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7431 kDefaultFramerate);
7432 // Capture larger frame to trigger a reconfigure.
7433 codec_height_ *= 2;
7434 codec_width_ *= 2;
7435 timestamp_ms += 1000 / kDefaultFramerate;
7436 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7437 WaitForEncodedFrame(timestamp_ms);
7438
7439 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7440 auto current_rate_settings =
7441 fake_encoder_.GetAndResetLastRateControlSettings();
7442 // Ensure we have actually reconfigured twice
7443 // The rate settings should have been set again even though
7444 // they haven't changed.
7445 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 07:55:037446 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 10:50:237447
7448 video_stream_encoder_->Stop();
7449}
7450
philipeld9cc8c02019-09-16 12:53:407451struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 17:17:517452 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7453 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7454 MOCK_METHOD(void,
7455 RequestEncoderSwitch,
7456 (const webrtc::SdpVideoFormat& format),
7457 (override));
philipeld9cc8c02019-09-16 12:53:407458};
7459
philipel9b058032020-02-10 10:30:007460TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7461 constexpr int kDontCare = 100;
7462 StrictMock<MockEncoderSelector> encoder_selector;
7463 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7464 &fake_encoder_, &encoder_selector);
7465 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7466
7467 // Reset encoder for new configuration to take effect.
7468 ConfigureEncoder(video_encoder_config_.Copy());
7469
7470 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7471
7472 video_source_.IncomingCapturedFrame(
7473 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 09:11:557474 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 10:30:007475 video_stream_encoder_->Stop();
7476
7477 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7478 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 14:22:117479 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7480 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 10:30:007481 video_stream_encoder_.reset();
7482}
7483
7484TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7485 constexpr int kDontCare = 100;
7486
7487 NiceMock<MockEncoderSelector> encoder_selector;
7488 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7489 video_send_config_.encoder_settings.encoder_switch_request_callback =
7490 &switch_callback;
7491 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7492 &fake_encoder_, &encoder_selector);
7493 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7494
7495 // Reset encoder for new configuration to take effect.
7496 ConfigureEncoder(video_encoder_config_.Copy());
7497
Mirta Dvornicic4f34d782020-02-26 12:01:197498 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 10:30:007499 .WillByDefault(Return(SdpVideoFormat("AV1")));
7500 EXPECT_CALL(switch_callback,
7501 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7502 Field(&SdpVideoFormat::name, "AV1"))));
7503
Henrik Boström381d1092020-05-12 16:49:077504 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077505 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7506 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7507 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 10:30:007508 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487509 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 10:30:007510 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 09:11:557511 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 10:30:007512
7513 video_stream_encoder_->Stop();
7514}
7515
7516TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7517 constexpr int kSufficientBitrateToNotDrop = 1000;
7518 constexpr int kDontCare = 100;
7519
7520 NiceMock<MockVideoEncoder> video_encoder;
7521 NiceMock<MockEncoderSelector> encoder_selector;
7522 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7523 video_send_config_.encoder_settings.encoder_switch_request_callback =
7524 &switch_callback;
7525 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7526 &video_encoder, &encoder_selector);
7527 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7528
7529 // Reset encoder for new configuration to take effect.
7530 ConfigureEncoder(video_encoder_config_.Copy());
7531
7532 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7533 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7534 // not fail.
Henrik Boström381d1092020-05-12 16:49:077535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077536 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7537 /*stable_target_bitrate=*/
7538 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7539 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 10:30:007540 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487541 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 10:30:007542 /*cwnd_reduce_ratio=*/0);
7543
7544 ON_CALL(video_encoder, Encode(_, _))
7545 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7546 ON_CALL(encoder_selector, OnEncoderBroken())
7547 .WillByDefault(Return(SdpVideoFormat("AV2")));
7548
7549 rtc::Event encode_attempted;
7550 EXPECT_CALL(switch_callback,
7551 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7552 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7553 EXPECT_EQ(format.name, "AV2");
7554 encode_attempted.Set();
7555 });
7556
7557 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7558 encode_attempted.Wait(3000);
7559
Markus Handell28c71802021-11-08 09:11:557560 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 13:56:427561
philipel9b058032020-02-10 10:30:007562 video_stream_encoder_->Stop();
7563
7564 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7565 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 14:22:117566 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7567 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 10:30:007568 video_stream_encoder_.reset();
7569}
7570
Evan Shrubsole7c079f62019-09-26 07:55:037571TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117572 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037573 const int kFrameWidth = 320;
7574 const int kFrameHeight = 180;
7575
7576 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077577 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037579 /*target_bitrate=*/rate,
7580 /*stable_target_bitrate=*/rate,
7581 /*link_allocation=*/rate,
7582 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487583 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327584 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037585
7586 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237587 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037588 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7589 frame.set_rotation(kVideoRotation_270);
7590 video_source_.IncomingCapturedFrame(frame);
7591 WaitForEncodedFrame(timestamp_ms);
7592 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7593
7594 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 17:46:077595 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077596 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037597 /*target_bitrate=*/new_stable_rate,
7598 /*stable_target_bitrate=*/new_stable_rate,
7599 /*link_allocation=*/rate,
7600 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487601 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327602 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037603 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7604 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7605 video_stream_encoder_->Stop();
7606}
7607
7608TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117609 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037610 const int kFrameWidth = 320;
7611 const int kFrameHeight = 180;
7612
7613 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077614 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037616 /*target_bitrate=*/rate,
7617 /*stable_target_bitrate=*/rate,
7618 /*link_allocation=*/rate,
7619 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487620 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327621 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037622
7623 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237624 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037625 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7626 frame.set_rotation(kVideoRotation_270);
7627 video_source_.IncomingCapturedFrame(frame);
7628 WaitForEncodedFrame(timestamp_ms);
7629 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7630
7631 // Set a higher target rate without changing the link_allocation. Should not
7632 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077633 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077634 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037635 /*target_bitrate=*/rate,
7636 /*stable_target_bitrate=*/new_stable_rate,
7637 /*link_allocation=*/rate,
7638 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487639 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327640 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037641 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7642 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7643 video_stream_encoder_->Stop();
7644}
7645
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177646TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7647 test::ScopedFieldTrials field_trials(
7648 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7649 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7650 const int kFramerateFps = 30;
7651 const int kWidth = 1920;
7652 const int kHeight = 1080;
7653 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7654 // Works on screenshare mode.
7655 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7656 // We rely on the automatic resolution adaptation, but we handle framerate
7657 // adaptation manually by mocking the stats proxy.
7658 video_source_.set_adaptation_enabled(true);
7659
7660 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 16:49:077661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117662 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177663 video_stream_encoder_->SetSource(&video_source_,
7664 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:397665 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177666
7667 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7668 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7669
7670 // Pass enough frames with the full update to trigger animation detection.
7671 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237672 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177673 frame.set_ntp_time_ms(timestamp_ms);
7674 frame.set_timestamp_us(timestamp_ms * 1000);
7675 video_source_.IncomingCapturedFrame(frame);
7676 WaitForEncodedFrame(timestamp_ms);
7677 }
7678
7679 // Resolution should be limited.
7680 rtc::VideoSinkWants expected;
7681 expected.max_framerate_fps = kFramerateFps;
7682 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 14:19:547683 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177684
7685 // Pass one frame with no known update.
7686 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 12:31:237687 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177688 frame.set_ntp_time_ms(timestamp_ms);
7689 frame.set_timestamp_us(timestamp_ms * 1000);
7690 frame.clear_update_rect();
7691
7692 video_source_.IncomingCapturedFrame(frame);
7693 WaitForEncodedFrame(timestamp_ms);
7694
7695 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 12:08:397696 EXPECT_THAT(video_source_.sink_wants(),
7697 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177698
7699 video_stream_encoder_->Stop();
7700}
7701
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327702TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7703 const int kWidth = 720; // 540p adapted down.
7704 const int kHeight = 405;
7705 const int kNumFrames = 3;
7706 // Works on screenshare mode.
7707 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7708 /*num_spatial_layers=*/2, /*screenshare=*/true);
7709
7710 video_source_.set_adaptation_enabled(true);
7711
7712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117713 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327714
7715 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7716
7717 // Pass enough frames with the full update to trigger animation detection.
7718 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237719 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327720 frame.set_ntp_time_ms(timestamp_ms);
7721 frame.set_timestamp_us(timestamp_ms * 1000);
7722 video_source_.IncomingCapturedFrame(frame);
7723 WaitForEncodedFrame(timestamp_ms);
7724 }
7725
7726 video_stream_encoder_->Stop();
7727}
7728
Yun Zhang1e4d4fd2020-09-30 07:22:467729TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7730 const float downscale_factors[] = {4.0, 2.0, 1.0};
7731 const int number_layers =
7732 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7733 VideoEncoderConfig config;
7734 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7735 for (int i = 0; i < number_layers; ++i) {
7736 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7737 config.simulcast_layers[i].active = true;
7738 }
7739 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:437740 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 07:22:467741 "VP8", /*max qp*/ 56, /*screencast*/ false,
7742 /*screenshare enabled*/ false);
7743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117744 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7745 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 07:22:467746
7747 // First initialization.
7748 // Encoder should be initialized. Next frame should be key frame.
7749 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7750 sink_.SetNumExpectedLayers(number_layers);
7751 int64_t timestamp_ms = kFrameIntervalMs;
7752 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7753 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117754 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467755 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7756 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7757 VideoFrameType::kVideoFrameKey,
7758 VideoFrameType::kVideoFrameKey}));
7759
7760 // Disable top layer.
7761 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7762 config.simulcast_layers[number_layers - 1].active = false;
7763 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7764 sink_.SetNumExpectedLayers(number_layers - 1);
7765 timestamp_ms += kFrameIntervalMs;
7766 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7767 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117768 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467769 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7770 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7771 VideoFrameType::kVideoFrameDelta,
7772 VideoFrameType::kVideoFrameDelta}));
7773
7774 // Re-enable top layer.
7775 // Encoder should be re-initialized. Next frame should be key frame.
7776 config.simulcast_layers[number_layers - 1].active = true;
7777 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7778 sink_.SetNumExpectedLayers(number_layers);
7779 timestamp_ms += kFrameIntervalMs;
7780 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7781 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117782 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467783 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7784 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7785 VideoFrameType::kVideoFrameKey,
7786 VideoFrameType::kVideoFrameKey}));
7787
7788 // Top layer max rate change.
7789 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7790 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7791 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7792 sink_.SetNumExpectedLayers(number_layers);
7793 timestamp_ms += kFrameIntervalMs;
7794 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7795 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117796 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467797 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7798 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7799 VideoFrameType::kVideoFrameDelta,
7800 VideoFrameType::kVideoFrameDelta}));
7801
7802 // Top layer resolution change.
7803 // Encoder should be re-initialized. Next frame should be key frame.
7804 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7805 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7806 sink_.SetNumExpectedLayers(number_layers);
7807 timestamp_ms += kFrameIntervalMs;
7808 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7809 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117810 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467811 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7812 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7813 VideoFrameType::kVideoFrameKey,
7814 VideoFrameType::kVideoFrameKey}));
7815 video_stream_encoder_->Stop();
7816}
7817
Henrik Boström1124ed12021-02-25 09:30:397818TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7819 const int kFrameWidth = 1280;
7820 const int kFrameHeight = 720;
7821
7822 SetUp();
7823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 09:30:397825
7826 // Capturing a frame should reconfigure the encoder and expose the encoder
7827 // resolution, which is the same as the input frame.
7828 int64_t timestamp_ms = kFrameIntervalMs;
7829 video_source_.IncomingCapturedFrame(
7830 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7831 WaitForEncodedFrame(timestamp_ms);
7832 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7833 EXPECT_THAT(video_source_.sink_wants().resolutions,
7834 ::testing::ElementsAreArray(
7835 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7836
7837 video_stream_encoder_->Stop();
7838}
7839
7840TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7841 // Pick downscale factors such that we never encode at full resolution - this
7842 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 14:22:117843 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 09:30:397844 // encoder should not ask for the frame resolution. This allows video frames
7845 // to have the appearence of one resolution but optimize its internal buffers
7846 // for what is actually encoded.
7847 const size_t kNumSimulcastLayers = 3u;
7848 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7849 const int kFrameWidth = 1280;
7850 const int kFrameHeight = 720;
7851 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7852 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7853 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7854 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7855 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7856 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7857
7858 VideoEncoderConfig config;
7859 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7860 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7861 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7862 config.simulcast_layers[i].active = true;
7863 }
7864 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:437865 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 09:30:397866 "VP8", /*max qp*/ 56, /*screencast*/ false,
7867 /*screenshare enabled*/ false);
7868 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117869 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7870 0, 0, 0);
Henrik Boström1124ed12021-02-25 09:30:397871
7872 // Capture a frame with all layers active.
7873 int64_t timestamp_ms = kFrameIntervalMs;
7874 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7875 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7876 video_source_.IncomingCapturedFrame(
7877 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7878 WaitForEncodedFrame(timestamp_ms);
7879 // Expect encoded resolutions to match the expected simulcast layers.
7880 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7881 EXPECT_THAT(
7882 video_source_.sink_wants().resolutions,
7883 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7884
7885 // Capture a frame with one of the layers inactive.
7886 timestamp_ms += kFrameIntervalMs;
7887 config.simulcast_layers[2].active = false;
7888 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7889 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7890 video_source_.IncomingCapturedFrame(
7891 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7892 WaitForEncodedFrame(timestamp_ms);
7893
7894 // Expect encoded resolutions to match the expected simulcast layers.
7895 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7896 EXPECT_THAT(video_source_.sink_wants().resolutions,
7897 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7898
7899 // Capture a frame with all but one layer turned off.
7900 timestamp_ms += kFrameIntervalMs;
7901 config.simulcast_layers[1].active = false;
7902 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7903 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7904 video_source_.IncomingCapturedFrame(
7905 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7906 WaitForEncodedFrame(timestamp_ms);
7907
7908 // Expect encoded resolutions to match the expected simulcast layers.
7909 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7910 EXPECT_THAT(video_source_.sink_wants().resolutions,
7911 ::testing::ElementsAreArray({kLayer0Size}));
7912
7913 video_stream_encoder_->Stop();
7914}
7915
Sergey Silkin0e42cf72021-03-15 09:12:577916TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 09:12:577917 ResetEncoder("VP8", 1, 1, 1, false);
7918
Niels Möller8b692902021-06-14 10:04:577919 // Force encoder reconfig.
7920 video_source_.IncomingCapturedFrame(
7921 CreateFrame(1, codec_width_, codec_height_));
7922 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7923
Sergey Silkin0e42cf72021-03-15 09:12:577924 // Set QP on encoded frame and pass the frame to encode complete callback.
7925 // Since QP is present QP parsing won't be triggered and the original value
7926 // should be kept.
7927 EncodedImage encoded_image;
7928 encoded_image.qp_ = 123;
7929 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7930 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7931 CodecSpecificInfo codec_info;
7932 codec_info.codecType = kVideoCodecVP8;
7933 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7934 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7935 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7936 video_stream_encoder_->Stop();
7937}
7938
7939TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 09:12:577940 ResetEncoder("VP8", 1, 1, 1, false);
7941
Niels Möller8b692902021-06-14 10:04:577942 // Force encoder reconfig.
7943 video_source_.IncomingCapturedFrame(
7944 CreateFrame(1, codec_width_, codec_height_));
7945 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7946
Sergey Silkin0e42cf72021-03-15 09:12:577947 // Pass an encoded frame without QP to encode complete callback. QP should be
7948 // parsed and set.
7949 EncodedImage encoded_image;
7950 encoded_image.qp_ = -1;
7951 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7952 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7953 CodecSpecificInfo codec_info;
7954 codec_info.codecType = kVideoCodecVP8;
7955 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7956 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7957 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7958 video_stream_encoder_->Stop();
7959}
7960
7961TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7962 webrtc::test::ScopedFieldTrials field_trials(
7963 "WebRTC-QpParsingKillSwitch/Enabled/");
7964
Sergey Silkin0e42cf72021-03-15 09:12:577965 ResetEncoder("VP8", 1, 1, 1, false);
7966
Niels Möller8b692902021-06-14 10:04:577967 // Force encoder reconfig.
7968 video_source_.IncomingCapturedFrame(
7969 CreateFrame(1, codec_width_, codec_height_));
7970 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7971
Sergey Silkin0e42cf72021-03-15 09:12:577972 EncodedImage encoded_image;
7973 encoded_image.qp_ = -1;
7974 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7975 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7976 CodecSpecificInfo codec_info;
7977 codec_info.codecType = kVideoCodecVP8;
7978 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7979 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7980 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7981 video_stream_encoder_->Stop();
7982}
7983
Sergey Silkind19e3b92021-03-16 10:05:307984TEST_F(VideoStreamEncoderTest,
7985 QualityScalingNotAllowed_QualityScalingDisabled) {
7986 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7987
7988 // Disable scaling settings in encoder info.
7989 fake_encoder_.SetQualityScaling(false);
7990 // Disable quality scaling in encoder config.
7991 video_encoder_config.is_quality_scaling_allowed = false;
7992 ConfigureEncoder(std::move(video_encoder_config));
7993
7994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117995 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:307996
7997 test::FrameForwarder source;
7998 video_stream_encoder_->SetSource(
7999 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8000 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8002
8003 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8004 WaitForEncodedFrame(1);
8005 video_stream_encoder_->TriggerQualityLow();
8006 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8007
8008 video_stream_encoder_->Stop();
8009}
8010
Qiu Jianlinb54cfde2021-07-29 22:48:038011TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8012 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8013
8014 // Disable scaling settings in encoder info.
8015 fake_encoder_.SetQualityScaling(false);
8016 // Set QP trusted in encoder info.
8017 fake_encoder_.SetIsQpTrusted(true);
8018 // Enable quality scaling in encoder config.
8019 video_encoder_config.is_quality_scaling_allowed = false;
8020 ConfigureEncoder(std::move(video_encoder_config));
8021
8022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118023 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-29 22:48:038024
8025 test::FrameForwarder source;
8026 video_stream_encoder_->SetSource(
8027 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8028 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8029 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8030
8031 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8032 WaitForEncodedFrame(1);
8033 video_stream_encoder_->TriggerQualityLow();
8034 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8035
8036 video_stream_encoder_->Stop();
8037}
8038
Shuhai Pengf2707702021-09-29 09:19:448039TEST_F(VideoStreamEncoderTest,
8040 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8041 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8042
8043 // Disable scaling settings in encoder info.
8044 fake_encoder_.SetQualityScaling(false);
8045 // Set QP trusted in encoder info.
8046 fake_encoder_.SetIsQpTrusted(true);
8047 // Enable quality scaling in encoder config.
8048 video_encoder_config.is_quality_scaling_allowed = false;
8049 ConfigureEncoder(std::move(video_encoder_config));
8050
8051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118052 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448053
8054 test::FrameForwarder source;
8055 video_stream_encoder_->SetSource(
8056 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8057 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8058 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8059
8060 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8061 WaitForEncodedFrame(1);
8062 video_stream_encoder_->TriggerQualityLow();
8063 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8064
8065 video_stream_encoder_->Stop();
8066}
8067
8068TEST_F(VideoStreamEncoderTest,
8069 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8070 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8071
8072 // Disable scaling settings in encoder info.
8073 fake_encoder_.SetQualityScaling(false);
8074 // Set QP trusted in encoder info.
8075 fake_encoder_.SetIsQpTrusted(false);
8076 // Enable quality scaling in encoder config.
8077 video_encoder_config.is_quality_scaling_allowed = false;
8078 ConfigureEncoder(std::move(video_encoder_config));
8079
8080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118081 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448082
8083 test::FrameForwarder source;
8084 video_stream_encoder_->SetSource(
8085 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8086 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8087 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8088
8089 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8090 WaitForEncodedFrame(1);
8091 video_stream_encoder_->TriggerQualityLow();
8092 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8093
8094 video_stream_encoder_->Stop();
8095}
8096
8097TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8098 // Set QP trusted in encoder info.
8099 fake_encoder_.SetIsQpTrusted(false);
8100
8101 const int MinEncBitrateKbps = 30;
8102 const int MaxEncBitrateKbps = 100;
8103 const int MinStartBitrateKbp = 50;
8104 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8105 /*frame_size_pixels=*/codec_width_ * codec_height_,
8106 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8107 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8108 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8109
8110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118111 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448112
8113 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8114
8115 VideoEncoderConfig video_encoder_config;
8116 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8117 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8118 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8119 MinEncBitrateKbps * 1000;
8120 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8121 kMaxPayloadLength);
8122
8123 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8124 WaitForEncodedFrame(1);
8125 EXPECT_EQ(
8126 MaxEncBitrateKbps,
8127 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8128 EXPECT_EQ(
8129 MinEncBitrateKbps,
8130 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8131
8132 video_stream_encoder_->Stop();
8133}
8134
8135TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8136 // Set QP trusted in encoder info.
8137 fake_encoder_.SetIsQpTrusted(false);
8138
8139 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8140 EncoderInfoSettings::
8141 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8142 codec_width_ * codec_height_,
8143 EncoderInfoSettings::
8144 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8145 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8146
8147 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8148 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8149 const int TargetEncBitrate = MaxEncBitrate;
8150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8151 DataRate::BitsPerSec(TargetEncBitrate),
8152 DataRate::BitsPerSec(TargetEncBitrate),
8153 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8154
8155 VideoEncoderConfig video_encoder_config;
8156 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8157 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8158 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8159 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8160 kMaxPayloadLength);
8161
8162 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8163 WaitForEncodedFrame(1);
8164 EXPECT_EQ(
8165 MaxEncBitrate / 1000,
8166 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8167 EXPECT_EQ(
8168 MinEncBitrate / 1000,
8169 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8170
8171 video_stream_encoder_->Stop();
8172}
8173
Sergey Silkind19e3b92021-03-16 10:05:308174#if !defined(WEBRTC_IOS)
8175// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8176// disabled by default on iOS.
8177TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8178 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8179
8180 // Disable scaling settings in encoder info.
8181 fake_encoder_.SetQualityScaling(false);
8182 // Enable quality scaling in encoder config.
8183 video_encoder_config.is_quality_scaling_allowed = true;
8184 ConfigureEncoder(std::move(video_encoder_config));
8185
8186 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118187 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:308188
8189 test::FrameForwarder source;
8190 video_stream_encoder_->SetSource(
8191 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8192 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8193 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8194
8195 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8196 WaitForEncodedFrame(1);
8197 video_stream_encoder_->TriggerQualityLow();
8198 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8199
8200 video_stream_encoder_->Stop();
8201}
Qiu Jianlinb54cfde2021-07-29 22:48:038202
8203TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8204 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8205
8206 // Disable scaling settings in encoder info.
8207 fake_encoder_.SetQualityScaling(false);
8208 // Set QP trusted in encoder info.
8209 fake_encoder_.SetIsQpTrusted(true);
8210 // Enable quality scaling in encoder config.
8211 video_encoder_config.is_quality_scaling_allowed = true;
8212 ConfigureEncoder(std::move(video_encoder_config));
8213
8214 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118215 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-29 22:48:038216
8217 test::FrameForwarder source;
8218 video_stream_encoder_->SetSource(
8219 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8220 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8221 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8222
8223 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8224 WaitForEncodedFrame(1);
8225 video_stream_encoder_->TriggerQualityLow();
8226 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8227
8228 video_stream_encoder_->Stop();
8229}
Shuhai Pengf2707702021-09-29 09:19:448230
8231TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8232 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8233
8234 // Disable scaling settings in encoder info.
8235 fake_encoder_.SetQualityScaling(false);
8236 // Set QP not trusted in encoder info.
8237 fake_encoder_.SetIsQpTrusted(false);
8238 // Enable quality scaling in encoder config.
8239 video_encoder_config.is_quality_scaling_allowed = true;
8240 ConfigureEncoder(std::move(video_encoder_config));
8241
8242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118243 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448244
8245 test::FrameForwarder source;
8246 video_stream_encoder_->SetSource(
8247 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8248 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8249 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8250
8251 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8252 WaitForEncodedFrame(1);
8253 video_stream_encoder_->TriggerQualityLow();
8254 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8255 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8256 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8257
8258 video_stream_encoder_->Stop();
8259}
8260
8261TEST_F(VideoStreamEncoderTest,
8262 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8263 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8264
8265 // Disable scaling settings in encoder info.
8266 fake_encoder_.SetQualityScaling(false);
8267 // Set QP trusted in encoder info.
8268 fake_encoder_.SetIsQpTrusted(true);
8269 // Enable quality scaling in encoder config.
8270 video_encoder_config.is_quality_scaling_allowed = true;
8271 ConfigureEncoder(std::move(video_encoder_config));
8272
8273 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118274 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448275
8276 test::FrameForwarder source;
8277 video_stream_encoder_->SetSource(
8278 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8279 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8280 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8281
8282 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8283 WaitForEncodedFrame(1);
8284 video_stream_encoder_->TriggerQualityLow();
8285 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8287
8288 video_stream_encoder_->Stop();
8289}
8290
8291TEST_F(VideoStreamEncoderTest,
8292 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8293 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8294
8295 // Disable scaling settings in encoder info.
8296 fake_encoder_.SetQualityScaling(false);
8297 // Set QP trusted in encoder info.
8298 fake_encoder_.SetIsQpTrusted(false);
8299 // Enable quality scaling in encoder config.
8300 video_encoder_config.is_quality_scaling_allowed = true;
8301 ConfigureEncoder(std::move(video_encoder_config));
8302
8303 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118304 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448305
8306 test::FrameForwarder source;
8307 video_stream_encoder_->SetSource(
8308 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8309 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8310 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8311
8312 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8313 WaitForEncodedFrame(1);
8314 video_stream_encoder_->TriggerQualityLow();
8315 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8316
8317 video_stream_encoder_->Stop();
8318}
8319
Sergey Silkind19e3b92021-03-16 10:05:308320#endif
8321
Henrik Boström56db9ff2021-03-24 08:06:458322// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8323class VideoStreamEncoderWithRealEncoderTest
8324 : public VideoStreamEncoderTest,
8325 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8326 public:
8327 VideoStreamEncoderWithRealEncoderTest()
8328 : VideoStreamEncoderTest(),
8329 codec_type_(std::get<0>(GetParam())),
8330 allow_i420_conversion_(std::get<1>(GetParam())) {}
8331
8332 void SetUp() override {
8333 VideoStreamEncoderTest::SetUp();
8334 std::unique_ptr<VideoEncoder> encoder;
8335 switch (codec_type_) {
8336 case kVideoCodecVP8:
8337 encoder = VP8Encoder::Create();
8338 break;
8339 case kVideoCodecVP9:
8340 encoder = VP9Encoder::Create();
8341 break;
8342 case kVideoCodecAV1:
8343 encoder = CreateLibaomAv1Encoder();
8344 break;
8345 case kVideoCodecH264:
8346 encoder =
8347 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8348 break;
8349 case kVideoCodecMultiplex:
8350 mock_encoder_factory_for_multiplex_ =
8351 std::make_unique<MockVideoEncoderFactory>();
8352 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8353 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8354 .WillRepeatedly([] { return VP8Encoder::Create(); });
8355 encoder = std::make_unique<MultiplexEncoderAdapter>(
8356 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8357 false);
8358 break;
8359 default:
Artem Titovd3251962021-11-15 15:57:078360 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 08:06:458361 }
8362 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8363 }
8364
8365 void TearDown() override {
8366 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 14:22:118367 // Ensure `video_stream_encoder_` is destroyed before
8368 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 08:06:458369 video_stream_encoder_.reset();
8370 VideoStreamEncoderTest::TearDown();
8371 }
8372
8373 protected:
8374 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8375 std::unique_ptr<VideoEncoder> encoder) {
8376 // Configure VSE to use the encoder.
8377 encoder_ = std::move(encoder);
8378 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8379 encoder_.get(), &encoder_selector_);
8380 video_send_config_.encoder_settings.encoder_factory =
8381 encoder_proxy_factory_.get();
8382 VideoEncoderConfig video_encoder_config;
8383 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8384 video_encoder_config_ = video_encoder_config.Copy();
8385 ConfigureEncoder(video_encoder_config_.Copy());
8386
8387 // Set bitrate to ensure frame is not dropped.
8388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118389 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 08:06:458390 }
8391
8392 const VideoCodecType codec_type_;
8393 const bool allow_i420_conversion_;
8394 NiceMock<MockEncoderSelector> encoder_selector_;
8395 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8396 std::unique_ptr<VideoEncoder> encoder_;
8397 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8398};
8399
8400TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8401 auto native_i420_frame = test::CreateMappableNativeFrame(
8402 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8403 video_source_.IncomingCapturedFrame(native_i420_frame);
8404 WaitForEncodedFrame(codec_width_, codec_height_);
8405
8406 auto mappable_native_buffer =
8407 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8408 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8409 mappable_native_buffer->GetMappedFramedBuffers();
8410 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8411 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8412 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8413 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8414}
8415
8416TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8417 auto native_nv12_frame = test::CreateMappableNativeFrame(
8418 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8419 video_source_.IncomingCapturedFrame(native_nv12_frame);
8420 WaitForEncodedFrame(codec_width_, codec_height_);
8421
8422 auto mappable_native_buffer =
8423 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8424 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8425 mappable_native_buffer->GetMappedFramedBuffers();
8426 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8427 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8428 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8429 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8430
8431 if (!allow_i420_conversion_) {
8432 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8433 }
8434}
8435
Erik Språng7444b192021-06-02 12:02:138436TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8437 if (codec_type_ == kVideoCodecMultiplex) {
8438 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8439 return;
8440 }
8441
8442 const size_t kNumSpatialLayers = 3u;
8443 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8444 const int kFrameWidth = 1280;
8445 const int kFrameHeight = 720;
8446 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8447 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8448 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8449 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8450 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8451 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8452
8453 VideoEncoderConfig config;
8454 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8455 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 08:07:118456 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 12:02:138457 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8458 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8459 vp9_settings.numberOfTemporalLayers = 3;
8460 vp9_settings.automaticResizeOn = false;
8461 config.encoder_specific_settings =
8462 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8463 vp9_settings);
8464 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8465 /*fps=*/30.0,
8466 /*first_active_layer=*/0,
8467 /*num_spatial_layers=*/3,
8468 /*num_temporal_layers=*/3,
8469 /*is_screenshare=*/false);
8470 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8471 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 08:07:118472 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 12:02:138473 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8474 /*fps=*/30.0,
8475 /*first_active_layer=*/0,
8476 /*num_spatial_layers=*/3,
8477 /*num_temporal_layers=*/3,
8478 /*is_screenshare=*/false);
8479 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8480 } else {
8481 // Simulcast for VP8/H264.
8482 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8483 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8484 config.simulcast_layers[i].scale_resolution_down_by =
8485 kDownscaleFactors[i];
8486 config.simulcast_layers[i].active = true;
8487 }
8488 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8489 // Turn off frame dropping to prevent flakiness.
8490 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8491 h264_settings.frameDroppingOn = false;
8492 config.encoder_specific_settings = rtc::make_ref_counted<
8493 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8494 }
8495 }
8496
8497 auto set_layer_active = [&](int layer_idx, bool active) {
8498 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8499 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8500 config.spatial_layers[layer_idx].active = active;
8501 } else {
8502 config.simulcast_layers[layer_idx].active = active;
8503 }
8504 };
8505
8506 config.video_stream_factory =
8507 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8508 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8509 /*screencast*/ false,
8510 /*screenshare enabled*/ false);
8511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118512 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8513 0, 0, 0);
Erik Språng7444b192021-06-02 12:02:138514
8515 // Capture a frame with all layers active.
8516 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8517 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8518 int64_t timestamp_ms = kFrameIntervalMs;
8519 video_source_.IncomingCapturedFrame(
8520 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8521
8522 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8523 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8524
8525 // Capture a frame with one of the layers inactive.
8526 set_layer_active(2, false);
8527 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8528 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8529 timestamp_ms += kFrameIntervalMs;
8530 video_source_.IncomingCapturedFrame(
8531 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8532 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8533
8534 // New target bitrates signaled based on lower resolution.
8535 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8536 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8537 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8538 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8539
8540 // Re-enable the top layer.
8541 set_layer_active(2, true);
8542 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8543 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8544 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8545
8546 // Bitrate target adjusted back up to enable HD layer...
8547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8548 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8549 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8550 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8551
8552 // ...then add a new frame.
8553 timestamp_ms += kFrameIntervalMs;
8554 video_source_.IncomingCapturedFrame(
8555 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8556 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8557 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8558
8559 video_stream_encoder_->Stop();
8560}
8561
Henrik Boström56db9ff2021-03-24 08:06:458562std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8563 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8564 VideoCodecType codec_type = std::get<0>(info.param);
8565 bool allow_i420_conversion = std::get<1>(info.param);
8566 std::string str;
8567 switch (codec_type) {
8568 case kVideoCodecGeneric:
8569 str = "Generic";
8570 break;
8571 case kVideoCodecVP8:
8572 str = "VP8";
8573 break;
8574 case kVideoCodecVP9:
8575 str = "VP9";
8576 break;
8577 case kVideoCodecAV1:
8578 str = "AV1";
8579 break;
8580 case kVideoCodecH264:
8581 str = "H264";
8582 break;
8583 case kVideoCodecMultiplex:
8584 str = "Multiplex";
8585 break;
8586 default:
Artem Titovd3251962021-11-15 15:57:078587 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 08:06:458588 }
8589 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8590 return str;
8591}
8592
8593constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8594 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8595constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8596 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8597constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8598 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8599constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8600 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8601#if defined(WEBRTC_USE_H264)
8602constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8603 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8604
8605// The windows compiler does not tolerate #if statements inside the
8606// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8607// and without H264).
8608INSTANTIATE_TEST_SUITE_P(
8609 All,
8610 VideoStreamEncoderWithRealEncoderTest,
8611 ::testing::Values(kVP8DisallowConversion,
8612 kVP9DisallowConversion,
8613 kAV1AllowConversion,
8614 kMultiplexDisallowConversion,
8615 kH264AllowConversion),
8616 TestParametersVideoCodecAndAllowI420ConversionToString);
8617#else
8618INSTANTIATE_TEST_SUITE_P(
8619 All,
8620 VideoStreamEncoderWithRealEncoderTest,
8621 ::testing::Values(kVP8DisallowConversion,
8622 kVP9DisallowConversion,
8623 kAV1AllowConversion,
8624 kMultiplexDisallowConversion),
8625 TestParametersVideoCodecAndAllowI420ConversionToString);
8626#endif
8627
Åsa Persson4d4f62f2021-09-12 14:14:488628class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8629 protected:
8630 void RunTest(const std::vector<VideoStream>& configs,
8631 const int expected_num_init_encode) {
8632 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 08:07:118633 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 14:14:488634 InsertFrameAndWaitForEncoded();
8635 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8636 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 08:07:118637 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8638 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 14:14:488639
8640 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8641 ConfigureEncoder(configs[1]);
8642 InsertFrameAndWaitForEncoded();
8643 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8644 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 08:07:118645 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 14:14:488646 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 08:07:118647 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 14:14:488648
8649 video_stream_encoder_->Stop();
8650 }
8651
8652 void ConfigureEncoder(const VideoStream& stream) {
8653 VideoEncoderConfig config;
8654 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8655 config.max_bitrate_bps = stream.max_bitrate_bps;
8656 config.simulcast_layers[0] = stream;
8657 config.video_stream_factory =
8658 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8659 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8660 /*conference_mode=*/false);
8661 video_stream_encoder_->ConfigureEncoder(std::move(config),
8662 kMaxPayloadLength);
8663 }
8664
8665 void OnBitrateUpdated(DataRate bitrate) {
8666 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8667 bitrate, bitrate, bitrate, 0, 0, 0);
8668 }
8669
8670 void InsertFrameAndWaitForEncoded() {
8671 timestamp_ms_ += kFrameIntervalMs;
8672 video_source_.IncomingCapturedFrame(
8673 CreateFrame(timestamp_ms_, kWidth, kHeight));
8674 sink_.WaitForEncodedFrame(timestamp_ms_);
8675 }
8676
8677 void ExpectEqual(const VideoCodec& actual,
8678 const VideoStream& expected) const {
8679 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8680 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8681 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8682 static_cast<unsigned int>(expected.min_bitrate_bps));
8683 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8684 static_cast<unsigned int>(expected.max_bitrate_bps));
8685 EXPECT_EQ(actual.simulcastStream[0].width,
8686 kWidth / expected.scale_resolution_down_by);
8687 EXPECT_EQ(actual.simulcastStream[0].height,
8688 kHeight / expected.scale_resolution_down_by);
8689 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8690 expected.num_temporal_layers);
8691 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8692 }
8693
8694 VideoStream DefaultConfig() const {
8695 VideoStream stream;
8696 stream.max_framerate = 25;
8697 stream.min_bitrate_bps = 35000;
8698 stream.max_bitrate_bps = 900000;
8699 stream.scale_resolution_down_by = 1.0;
8700 stream.num_temporal_layers = 1;
8701 stream.bitrate_priority = 1.0;
8702 stream.scalability_mode = "";
8703 return stream;
8704 }
8705
8706 const int kWidth = 640;
8707 const int kHeight = 360;
8708 int64_t timestamp_ms_ = 0;
8709};
8710
8711TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8712 VideoStream config1 = DefaultConfig();
8713 VideoStream config2 = config1;
8714 config2.max_framerate++;
8715
8716 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8717}
8718
8719TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8720 VideoStream config1 = DefaultConfig();
8721 VideoStream config2 = config1;
8722 config2.min_bitrate_bps += 10000;
8723
8724 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8725}
8726
8727TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8728 VideoStream config1 = DefaultConfig();
8729 VideoStream config2 = config1;
8730 config2.max_bitrate_bps += 100000;
8731
8732 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8733}
8734
8735TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8736 VideoStream config1 = DefaultConfig();
8737 VideoStream config2 = config1;
8738 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8739
8740 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8741}
8742
8743TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8744 VideoStream config1 = DefaultConfig();
8745 VideoStream config2 = config1;
8746 config2.scale_resolution_down_by *= 2;
8747
8748 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8749}
8750
8751TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8752 VideoStream config1 = DefaultConfig();
8753 VideoStream config2 = config1;
8754 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8755
8756 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8757}
8758
8759TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8760 VideoStream config1 = DefaultConfig();
8761 VideoStream config2 = config1;
8762 config2.scalability_mode = "L1T2";
8763
8764 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8765}
8766
Markus Handellb4e96d42021-11-05 11:00:558767TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8768 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8769 auto* adapter_ptr = adapter.get();
8770 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 10:37:168771 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8772 nullptr;
8773 EXPECT_CALL(*adapter_ptr, Initialize)
8774 .WillOnce(Invoke([&video_stream_encoder_callback](
8775 FrameCadenceAdapterInterface::Callback* callback) {
8776 video_stream_encoder_callback = callback;
8777 }));
8778 TaskQueueBase* encoder_queue = nullptr;
8779 auto video_stream_encoder =
8780 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 11:00:558781
Markus Handelle59fee82021-12-23 08:29:238782 // First a call before we know the frame size and hence cannot compute the
8783 // number of simulcast layers.
8784 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8785 &FrameCadenceAdapterInterface::
8786 ZeroHertzModeParams::num_simulcast_layers,
8787 Eq(0)))));
Markus Handellb4e96d42021-11-05 11:00:558788 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 10:37:168789 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 11:00:558790 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8791 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 08:29:238792 factory.DepleteTaskQueues();
8793
8794 // Then a call as we've computed the number of simulcast layers after a passed
8795 // frame.
8796 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8797 &FrameCadenceAdapterInterface::
8798 ZeroHertzModeParams::num_simulcast_layers,
8799 Gt(0)))));
Markus Handell8d87c462021-12-16 10:37:168800 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 15:07:018801 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 11:00:558802 Mock::VerifyAndClearExpectations(adapter_ptr);
8803
Markus Handelle59fee82021-12-23 08:29:238804 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 10:37:168805 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 11:00:558806 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 10:37:168807 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 11:00:558808 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8809 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 10:37:168810 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 15:07:018811 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 11:00:558812}
8813
8814TEST(VideoStreamEncoderFrameCadenceTest,
8815 ForwardsFramesIntoFrameCadenceAdapter) {
8816 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8817 auto* adapter_ptr = adapter.get();
8818 test::FrameForwarder video_source;
8819 SimpleVideoStreamEncoderFactory factory;
8820 auto video_stream_encoder = factory.Create(std::move(adapter));
8821 video_stream_encoder->SetSource(
8822 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8823
8824 EXPECT_CALL(*adapter_ptr, OnFrame);
8825 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8826 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 10:37:168827 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 11:00:558828}
8829
Markus Handellee225432021-11-29 11:35:128830TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
8831 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8832 auto* adapter_ptr = adapter.get();
8833 test::FrameForwarder video_source;
8834 SimpleVideoStreamEncoderFactory factory;
8835 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8836 nullptr;
8837 EXPECT_CALL(*adapter_ptr, Initialize)
8838 .WillOnce(Invoke([&video_stream_encoder_callback](
8839 FrameCadenceAdapterInterface::Callback* callback) {
8840 video_stream_encoder_callback = callback;
8841 }));
8842 TaskQueueBase* encoder_queue = nullptr;
8843 auto video_stream_encoder =
8844 factory.Create(std::move(adapter), &encoder_queue);
8845
8846 // This is just to make the VSE operational. We'll feed a frame directly by
8847 // the callback interface.
8848 video_stream_encoder->SetSource(
8849 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8850
8851 VideoEncoderConfig video_encoder_config;
8852 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
8853 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
8854 /*max_data_payload_length=*/1000);
8855
8856 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
8857 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 10:37:168858 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 11:35:128859 factory.DepleteTaskQueues();
8860}
8861
Markus Handell8d87c462021-12-16 10:37:168862TEST(VideoStreamEncoderFrameCadenceTest,
8863 DeactivatesActivatesLayersOnBitrateChanges) {
8864 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8865 auto* adapter_ptr = adapter.get();
8866 SimpleVideoStreamEncoderFactory factory;
8867 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8868 nullptr;
8869 EXPECT_CALL(*adapter_ptr, Initialize)
8870 .WillOnce(Invoke([&video_stream_encoder_callback](
8871 FrameCadenceAdapterInterface::Callback* callback) {
8872 video_stream_encoder_callback = callback;
8873 }));
8874 TaskQueueBase* encoder_queue = nullptr;
8875 auto video_stream_encoder =
8876 factory.Create(std::move(adapter), &encoder_queue);
8877
8878 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
8879 // {150000, 450000}.
8880 VideoEncoderConfig video_encoder_config;
8881 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
8882 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
8883 kMaxPayloadLength);
8884 // Ensure an encoder is created.
8885 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
8886
8887 // Both layers enabled at 1 MBit/s.
8888 video_stream_encoder->OnBitrateUpdated(
8889 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8890 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8891 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8892 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
8893 factory.DepleteTaskQueues();
8894 Mock::VerifyAndClearExpectations(adapter_ptr);
8895
8896 // Layer 1 disabled at 200 KBit/s.
8897 video_stream_encoder->OnBitrateUpdated(
8898 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
8899 DataRate::KilobitsPerSec(200), 0, 0, 0);
8900 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8901 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
8902 factory.DepleteTaskQueues();
8903 Mock::VerifyAndClearExpectations(adapter_ptr);
8904
8905 // All layers off at suspended video.
8906 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
8907 DataRate::Zero(), 0, 0, 0);
8908 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
8909 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
8910 factory.DepleteTaskQueues();
8911 Mock::VerifyAndClearExpectations(adapter_ptr);
8912
8913 // Both layers enabled again back at 1 MBit/s.
8914 video_stream_encoder->OnBitrateUpdated(
8915 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8916 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8917 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8918 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
8919 factory.DepleteTaskQueues();
8920}
8921
8922TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
8923 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8924 auto* adapter_ptr = adapter.get();
8925 SimpleVideoStreamEncoderFactory factory;
8926 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8927 nullptr;
8928 EXPECT_CALL(*adapter_ptr, Initialize)
8929 .WillOnce(Invoke([&video_stream_encoder_callback](
8930 FrameCadenceAdapterInterface::Callback* callback) {
8931 video_stream_encoder_callback = callback;
8932 }));
8933 TaskQueueBase* encoder_queue = nullptr;
8934 auto video_stream_encoder =
8935 factory.Create(std::move(adapter), &encoder_queue);
8936
8937 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
8938 VideoEncoderConfig video_encoder_config;
8939 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
8940 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
8941 kMaxPayloadLength);
8942 video_stream_encoder->OnBitrateUpdated(
8943 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8944 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8945
8946 // Pass a frame which has unconverged results.
8947 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
8948 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
8949 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
8950 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
8951 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
8952 CodecSpecificInfo codec_specific;
8953 codec_specific.codecType = kVideoCodecGeneric;
8954 return codec_specific;
8955 }));
8956 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
8957 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
8958 factory.DepleteTaskQueues();
8959 Mock::VerifyAndClearExpectations(adapter_ptr);
8960 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
8961
8962 // Pass a frame which converges in layer 0 and not in layer 1.
8963 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
8964 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
8965 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
8966 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
8967 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
8968 CodecSpecificInfo codec_specific;
8969 codec_specific.codecType = kVideoCodecGeneric;
8970 return codec_specific;
8971 }));
8972 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
8973 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
8974 factory.DepleteTaskQueues();
8975 Mock::VerifyAndClearExpectations(adapter_ptr);
8976 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
8977}
8978
Markus Handell2e0f4f02021-12-21 18:14:588979TEST(VideoStreamEncoderFrameCadenceTest,
8980 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
8981 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8982 auto* adapter_ptr = adapter.get();
8983 MockVideoSourceInterface mock_source;
8984 SimpleVideoStreamEncoderFactory factory;
8985 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8986 nullptr;
8987 EXPECT_CALL(*adapter_ptr, Initialize)
8988 .WillOnce(Invoke([&video_stream_encoder_callback](
8989 FrameCadenceAdapterInterface::Callback* callback) {
8990 video_stream_encoder_callback = callback;
8991 }));
8992 TaskQueueBase* encoder_queue = nullptr;
8993 auto video_stream_encoder =
8994 factory.Create(std::move(adapter), &encoder_queue);
8995 video_stream_encoder->SetSource(
8996 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8997 VideoEncoderConfig config;
8998 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8999 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9000 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9001 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9002 // Ensure the encoder is set up.
9003 factory.DepleteTaskQueues();
9004
9005 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest).WillOnce(Return(true));
9006 EXPECT_CALL(mock_source, RequestRefreshFrame);
9007 video_stream_encoder->SendKeyFrame();
9008 factory.DepleteTaskQueues();
9009 Mock::VerifyAndClearExpectations(adapter_ptr);
9010 Mock::VerifyAndClearExpectations(&mock_source);
9011
9012 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest).WillOnce(Return(false));
9013 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9014 video_stream_encoder->SendKeyFrame();
9015 factory.DepleteTaskQueues();
9016}
9017
perkj26091b12016-09-01 08:17:409018} // namespace webrtc