blob: aea1988df9f769989ede99db2f54a641222acf6e [file] [log] [blame]
perkj26091b12016-09-01 08:17:401/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 08:30:3111#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 15:24:5913#include <algorithm>
perkj803d97f2016-11-01 18:45:4614#include <limits>
Danil Chapovalovd3ba2362019-04-10 15:01:2315#include <memory>
Henrik Boström56db9ff2021-03-24 08:06:4516#include <tuple>
Per512ecb32016-09-23 13:52:0617#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 14:30:1619#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 15:07:0120#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 15:01:2321#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 12:01:3322#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 11:00:5523#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 09:20:0924#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 10:30:0025#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 08:06:4526#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 10:37:1627#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 18:02:5628#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3129#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 07:15:1330#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 09:36:5531#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 16:10:5732#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 08:06:4533#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 12:57:5734#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 08:30:3135#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 10:56:2036#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 11:02:3637#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 15:26:0138#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 14:30:1639#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 13:59:1240#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 17:11:0041#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 15:50:0042#include "media/engine/webrtc_video_engine.h"
philipel95701502022-01-18 17:47:5243#include "modules/video_coding/codecs/av1/libaom_av1_encoder_supported.h"
Henrik Boström56db9ff2021-03-24 08:06:4544#include "modules/video_coding/codecs/h264/include/h264.h"
45#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
46#include "modules/video_coding/codecs/vp8/include/vp8.h"
47#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 17:32:3748#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 12:02:1349#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 10:24:3350#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 11:06:5951#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2352#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 09:39:5153#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 14:29:0954#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3155#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 17:11:0056#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 11:13:3257#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 15:20:1758#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 06:51:1059#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3160#include "test/encoder_settings.h"
61#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 08:51:4062#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 14:59:0063#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3164#include "test/gmock.h"
65#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 08:06:4566#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2367#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 07:07:2468#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 11:00:5569#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3170#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 08:17:4071
72namespace webrtc {
73
sprang57c2fff2017-01-16 14:24:0274using ::testing::_;
philipeld9cc8c02019-09-16 12:53:4075using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 12:08:3976using ::testing::Eq;
philipeld9cc8c02019-09-16 12:53:4077using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 12:08:3978using ::testing::Ge;
79using ::testing::Gt;
Markus Handellee225432021-11-29 11:35:1280using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 12:08:3981using ::testing::Le;
82using ::testing::Lt;
philipel9b058032020-02-10 10:30:0083using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 11:00:5584using ::testing::Mock;
philipel9b058032020-02-10 10:30:0085using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 11:00:5586using ::testing::Optional;
philipel9b058032020-02-10 10:30:0087using ::testing::Return;
Per Kjellander4190ce92020-12-15 16:24:5588using ::testing::SizeIs;
philipeld9cc8c02019-09-16 12:53:4089using ::testing::StrictMock;
kthelgason876222f2016-11-29 09:44:1190
perkj803d97f2016-11-01 18:45:4691namespace {
Åsa Persson8c1bf952018-09-13 08:42:1992const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 14:56:0093const int kQpLow = 1;
94const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 08:42:1995const int kMinFramerateFps = 2;
96const int kMinBalancedFramerateFps = 7;
97const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 12:37:0098const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 08:07:1199const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
100const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
101const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
102const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 15:02:22103const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 11:21:07104const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 08:42:19105const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 08:48:48106const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 11:12:19107const VideoEncoder::ResolutionBitrateLimits
108 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
109const VideoEncoder::ResolutionBitrateLimits
110 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 12:37:00111
Asa Persson606d3cb2021-10-04 08:07:11112uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 14:30:16113 0x00, 0x00, 0x03, 0x03, 0xF4,
114 0x05, 0x03, 0xC7, 0xE0, 0x1B,
115 0x41, 0x10, 0x8D, 0x00};
116
Sergey Silkin0e42cf72021-03-15 09:12:57117const uint8_t kCodedFrameVp8Qp25[] = {
118 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
119 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
120 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
121
Markus Handell818e7fb2021-12-30 12:01:33122VideoFrame CreateSimpleNV12Frame() {
123 return VideoFrame::Builder()
124 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
125 /*width=*/16, /*height=*/16))
126 .build();
127}
128
Markus Handell8d87c462021-12-16 10:37:16129void PassAFrame(
130 TaskQueueBase* encoder_queue,
131 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
132 int64_t ntp_time_ms) {
133 encoder_queue->PostTask(
134 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
Markus Handell818e7fb2021-12-30 12:01:33135 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms),
136 1, CreateSimpleNV12Frame());
Markus Handell8d87c462021-12-16 10:37:16137 }));
138}
139
perkj803d97f2016-11-01 18:45:46140class TestBuffer : public webrtc::I420Buffer {
141 public:
142 TestBuffer(rtc::Event* event, int width, int height)
143 : I420Buffer(width, height), event_(event) {}
144
145 private:
146 friend class rtc::RefCountedObject<TestBuffer>;
147 ~TestBuffer() override {
148 if (event_)
149 event_->Set();
150 }
151 rtc::Event* const event_;
152};
153
Henrik Boström56db9ff2021-03-24 08:06:45154// A fake native buffer that can't be converted to I420. Upon scaling, it
155// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 13:59:12156class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
157 public:
158 FakeNativeBuffer(rtc::Event* event, int width, int height)
159 : event_(event), width_(width), height_(height) {}
160 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
161 int width() const override { return width_; }
162 int height() const override { return height_; }
163 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
164 return nullptr;
165 }
Henrik Boström56db9ff2021-03-24 08:06:45166 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
167 int offset_x,
168 int offset_y,
169 int crop_width,
170 int crop_height,
171 int scaled_width,
172 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 17:21:43173 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
174 scaled_height);
Henrik Boström56db9ff2021-03-24 08:06:45175 }
Noah Richards51db4212019-06-12 13:59:12176
177 private:
178 friend class rtc::RefCountedObject<FakeNativeBuffer>;
179 ~FakeNativeBuffer() override {
180 if (event_)
181 event_->Set();
182 }
183 rtc::Event* const event_;
184 const int width_;
185 const int height_;
186};
187
Evan Shrubsole895556e2020-10-05 07:15:13188// A fake native buffer that is backed by an NV12 buffer.
189class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
190 public:
191 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
192 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
193
194 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
195 int width() const override { return nv12_buffer_->width(); }
196 int height() const override { return nv12_buffer_->height(); }
197 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
198 return nv12_buffer_->ToI420();
199 }
Evan Shrubsoleb556b082020-10-08 12:56:45200 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
201 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
202 if (absl::c_find(types, Type::kNV12) != types.end()) {
203 return nv12_buffer_;
204 }
205 return nullptr;
206 }
Evan Shrubsole895556e2020-10-05 07:15:13207 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
208
209 private:
210 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
211 ~FakeNV12NativeBuffer() override {
212 if (event_)
213 event_->Set();
214 }
215 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
216 rtc::Event* const event_;
217};
218
Niels Möller7dc26b72017-12-06 09:27:48219class CpuOveruseDetectorProxy : public OveruseFrameDetector {
220 public:
Niels Möllerd1f7eb62018-03-28 14:40:58221 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
222 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 16:49:07223 last_target_framerate_fps_(-1),
224 framerate_updated_event_(true /* manual_reset */,
225 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 09:27:48226 virtual ~CpuOveruseDetectorProxy() {}
227
228 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 11:13:32229 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48230 last_target_framerate_fps_ = framerate_fps;
231 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 16:49:07232 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 09:27:48233 }
234
235 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 11:13:32236 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48237 return last_target_framerate_fps_;
238 }
239
Niels Möller4db138e2018-04-19 07:04:13240 CpuOveruseOptions GetOptions() { return options_; }
241
Henrik Boström381d1092020-05-12 16:49:07242 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
243
Niels Möller7dc26b72017-12-06 09:27:48244 private:
Markus Handella3765182020-07-08 11:13:32245 Mutex lock_;
Niels Möller7dc26b72017-12-06 09:27:48246 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 16:49:07247 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 09:27:48248};
249
Henrik Boström0f0aa9c2020-06-02 11:02:36250class FakeVideoSourceRestrictionsListener
251 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 16:49:07252 public:
Henrik Boström0f0aa9c2020-06-02 11:02:36253 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 16:49:07254 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 11:02:36255 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 16:49:07256 RTC_DCHECK(was_restrictions_updated_);
257 }
258
259 rtc::Event* restrictions_updated_event() {
260 return &restrictions_updated_event_;
261 }
262
Henrik Boström0f0aa9c2020-06-02 11:02:36263 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 16:49:07264 void OnVideoSourceRestrictionsUpdated(
265 VideoSourceRestrictions restrictions,
266 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 09:47:46267 rtc::scoped_refptr<Resource> reason,
268 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 16:49:07269 was_restrictions_updated_ = true;
270 restrictions_updated_event_.Set();
271 }
272
273 private:
274 bool was_restrictions_updated_;
275 rtc::Event restrictions_updated_event_;
276};
277
Evan Shrubsole5fd40602020-05-25 14:19:54278auto WantsFps(Matcher<int> fps_matcher) {
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_matcher);
281}
282
283auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
284 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
285 AllOf(max_pixel_matcher, Gt(0)));
286}
287
Evan Shrubsole5cd7eb82020-05-25 12:08:39288auto ResolutionMax() {
289 return AllOf(
Evan Shrubsole5fd40602020-05-25 14:19:54290 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 12:08:39291 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
292 Eq(absl::nullopt)));
293}
294
295auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 14:19:54296 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 12:08:39297}
298
299auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 14:19:54300 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 12:08:39301}
302
303auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54304 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 12:08:39305}
306
307auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54308 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 12:08:39309}
310
311auto FpsMaxResolutionMax() {
312 return AllOf(FpsMax(), ResolutionMax());
313}
314
315auto UnlimitedSinkWants() {
316 return AllOf(FpsUnlimited(), ResolutionMax());
317}
318
319auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
320 Matcher<int> fps_range_matcher;
321
322 if (last_frame_pixels <= 320 * 240) {
323 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 15:50:00324 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 12:08:39325 fps_range_matcher = AllOf(Ge(10), Le(15));
326 } else if (last_frame_pixels <= 640 * 480) {
327 fps_range_matcher = Ge(15);
328 } else {
329 fps_range_matcher = Eq(kDefaultFramerate);
330 }
331 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
332 fps_range_matcher);
333}
334
Evan Shrubsole5fd40602020-05-25 14:19:54335auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
336 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
337 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
338}
339
340auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
341 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
342}
343
344auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
345 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
346}
347
348auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
349 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
350 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
351}
352
353auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
354 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
355 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
356}
357
358auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
359 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
360 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
361}
362
363auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
364 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
365 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
366}
367
mflodmancc3d4422017-08-03 15:27:51368class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 18:45:46369 public:
Markus Handell9a478b52021-11-18 15:07:01370 VideoStreamEncoderUnderTest(
371 TimeController* time_controller,
372 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
373 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
374 encoder_queue,
375 SendStatisticsProxy* stats_proxy,
376 const VideoStreamEncoderSettings& settings,
377 VideoStreamEncoder::BitrateAllocationCallbackType
378 allocation_callback_type)
379 : VideoStreamEncoder(time_controller->GetClock(),
380 1 /* number_of_cores */,
381 stats_proxy,
382 settings,
383 std::unique_ptr<OveruseFrameDetector>(
384 overuse_detector_proxy_ =
385 new CpuOveruseDetectorProxy(stats_proxy)),
386 std::move(cadence_adapter),
387 std::move(encoder_queue),
388 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 12:31:23389 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 15:59:05390 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 11:02:36391 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 09:47:10392 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 14:29:22393 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 09:36:55394 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 14:29:22395 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 11:02:36396 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 15:26:01397 }
perkj803d97f2016-11-01 18:45:46398
Henrik Boström381d1092020-05-12 16:49:07399 void SetSourceAndWaitForRestrictionsUpdated(
400 rtc::VideoSourceInterface<VideoFrame>* source,
401 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 11:02:36402 FakeVideoSourceRestrictionsListener listener;
403 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07404 SetSource(source, degradation_preference);
405 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 11:02:36406 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07407 }
408
409 void SetSourceAndWaitForFramerateUpdated(
410 rtc::VideoSourceInterface<VideoFrame>* source,
411 const DegradationPreference& degradation_preference) {
412 overuse_detector_proxy_->framerate_updated_event()->Reset();
413 SetSource(source, degradation_preference);
414 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
415 }
416
417 void OnBitrateUpdatedAndWaitForManagedResources(
418 DataRate target_bitrate,
419 DataRate stable_target_bitrate,
420 DataRate link_allocation,
421 uint8_t fraction_lost,
422 int64_t round_trip_time_ms,
423 double cwnd_reduce_ratio) {
424 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
425 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
426 // Bitrate is updated on the encoder queue.
427 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 16:49:07428 }
429
kthelgason2fc52542017-03-03 08:24:41430 // This is used as a synchronisation mechanism, to make sure that the
431 // encoder queue is not blocked before we start sending it frames.
432 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 09:11:55433 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 08:24:41434 }
435
Henrik Boström91aa7322020-04-28 10:24:33436 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 13:25:32437 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 10:24:33438 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12439 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36440 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33441 event.Set();
442 });
443 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55444 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 10:24:33445 }
Tomas Gunnarsson612445e2020-09-21 12:31:23446
Henrik Boström91aa7322020-04-28 10:24:33447 void TriggerCpuUnderuse() {
448 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12449 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36450 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33451 event.Set();
452 });
453 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55454 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 13:25:32455 }
kthelgason876222f2016-11-29 09:44:11456
Henrik Boström91aa7322020-04-28 10:24:33457 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 13:25:32458 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 10:24:33459 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12460 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36461 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33462 event.Set();
463 });
464 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55465 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 13:25:32466 }
Åsa Perssonb67c44c2019-09-24 13:25:32467 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 10:24:33468 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12469 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36470 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33471 event.Set();
472 });
473 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 09:11:55474 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 10:24:33475 }
476
Tomas Gunnarsson612445e2020-09-21 12:31:23477 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 09:27:48478 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 14:29:22479 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
480 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 11:02:36481 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 18:45:46482};
483
Noah Richards51db4212019-06-12 13:59:12484// Simulates simulcast behavior and makes highest stream resolutions divisible
485// by 4.
486class CroppingVideoStreamFactory
487 : public VideoEncoderConfig::VideoStreamFactoryInterface {
488 public:
Åsa Persson17b29b92020-10-17 10:57:58489 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 13:59:12490
491 private:
492 std::vector<VideoStream> CreateEncoderStreams(
493 int width,
494 int height,
495 const VideoEncoderConfig& encoder_config) override {
496 std::vector<VideoStream> streams = test::CreateVideoStreams(
497 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 13:59:12498 return streams;
499 }
Noah Richards51db4212019-06-12 13:59:12500};
501
sprangb1ca0732017-02-01 16:38:12502class AdaptingFrameForwarder : public test::FrameForwarder {
503 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23504 explicit AdaptingFrameForwarder(TimeController* time_controller)
505 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 12:51:49506 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 16:38:12507
508 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 11:13:32509 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12510 adaptation_enabled_ = enabled;
511 }
512
asaperssonfab67072017-04-04 12:51:49513 bool adaption_enabled() const {
Markus Handella3765182020-07-08 11:13:32514 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12515 return adaptation_enabled_;
516 }
517
Henrik Boström1124ed12021-02-25 09:30:39518 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
519 // the resolution or frame rate was different than it is currently. If
520 // something else is modified, such as encoder resolutions, but the resolution
521 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-16 06:40:18522 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 11:13:32523 MutexLock lock(&mutex_);
asapersson09f05612017-05-16 06:40:18524 return last_wants_;
525 }
526
Danil Chapovalovb9b146c2018-06-15 10:28:07527 absl::optional<int> last_sent_width() const { return last_width_; }
528 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-09 01:04:29529
sprangb1ca0732017-02-01 16:38:12530 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 12:31:23531 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 09:11:55532 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:23533
sprangb1ca0732017-02-01 16:38:12534 int cropped_width = 0;
535 int cropped_height = 0;
536 int out_width = 0;
537 int out_height = 0;
sprangc5d62e22017-04-03 06:53:04538 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23539 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
540 << "w=" << video_frame.width()
541 << "h=" << video_frame.height();
sprangc5d62e22017-04-03 06:53:04542 if (adapter_.AdaptFrameResolution(
543 video_frame.width(), video_frame.height(),
544 video_frame.timestamp_us() * 1000, &cropped_width,
545 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 22:49:37546 VideoFrame adapted_frame =
547 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43548 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 22:49:37549 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 17:05:30550 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 22:49:37551 .set_timestamp_ms(99)
552 .set_rotation(kVideoRotation_0)
553 .build();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:17554 if (video_frame.has_update_rect()) {
555 adapted_frame.set_update_rect(
556 video_frame.update_rect().ScaleWithFrame(
557 video_frame.width(), video_frame.height(), 0, 0,
558 video_frame.width(), video_frame.height(), out_width,
559 out_height));
560 }
sprangc5d62e22017-04-03 06:53:04561 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-09 01:04:29562 last_width_.emplace(adapted_frame.width());
563 last_height_.emplace(adapted_frame.height());
564 } else {
Danil Chapovalovb9b146c2018-06-15 10:28:07565 last_width_ = absl::nullopt;
566 last_height_ = absl::nullopt;
sprangc5d62e22017-04-03 06:53:04567 }
sprangb1ca0732017-02-01 16:38:12568 } else {
Harald Alvestrand97597c02021-11-04 12:01:23569 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 16:38:12570 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-09 01:04:29571 last_width_.emplace(video_frame.width());
572 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 16:38:12573 }
574 }
575
Åsa Perssonf5f7e8e2021-06-09 20:55:38576 void OnOutputFormatRequest(int width, int height) {
577 absl::optional<std::pair<int, int>> target_aspect_ratio =
578 std::make_pair(width, height);
579 absl::optional<int> max_pixel_count = width * height;
580 absl::optional<int> max_fps;
581 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
582 max_fps);
583 }
584
sprangb1ca0732017-02-01 16:38:12585 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
586 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 11:13:32587 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 09:30:39588 rtc::VideoSinkWants prev_wants = sink_wants_locked();
589 bool did_adapt =
590 prev_wants.max_pixel_count != wants.max_pixel_count ||
591 prev_wants.target_pixel_count != wants.target_pixel_count ||
592 prev_wants.max_framerate_fps != wants.max_framerate_fps;
593 if (did_adapt) {
594 last_wants_ = prev_wants;
595 }
Rasmus Brandt287e4642019-11-15 15:56:01596 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 06:37:30597 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 16:38:12598 }
Tomas Gunnarsson612445e2020-09-21 12:31:23599
600 TimeController* const time_controller_;
sprangb1ca0732017-02-01 16:38:12601 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 11:13:32602 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
603 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 10:28:07604 absl::optional<int> last_width_;
605 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 16:38:12606};
sprangc5d62e22017-04-03 06:53:04607
Niels Möller213618e2018-07-24 07:29:58608// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-03 06:53:04609class MockableSendStatisticsProxy : public SendStatisticsProxy {
610 public:
611 MockableSendStatisticsProxy(Clock* clock,
612 const VideoSendStream::Config& config,
613 VideoEncoderConfig::ContentType content_type)
614 : SendStatisticsProxy(clock, config, content_type) {}
615
616 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 11:13:32617 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04618 if (mock_stats_)
619 return *mock_stats_;
620 return SendStatisticsProxy::GetStats();
621 }
622
Niels Möller213618e2018-07-24 07:29:58623 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 11:13:32624 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 07:29:58625 if (mock_stats_)
626 return mock_stats_->input_frame_rate;
627 return SendStatisticsProxy::GetInputFrameRate();
628 }
sprangc5d62e22017-04-03 06:53:04629 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 11:13:32630 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04631 mock_stats_.emplace(stats);
632 }
633
634 void ResetMockStats() {
Markus Handella3765182020-07-08 11:13:32635 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04636 mock_stats_.reset();
637 }
638
Tomas Gunnarsson612445e2020-09-21 12:31:23639 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
640 on_frame_dropped_ = std::move(callback);
641 }
642
sprangc5d62e22017-04-03 06:53:04643 private:
Tomas Gunnarsson612445e2020-09-21 12:31:23644 void OnFrameDropped(DropReason reason) override {
645 SendStatisticsProxy::OnFrameDropped(reason);
646 if (on_frame_dropped_)
647 on_frame_dropped_(reason);
648 }
649
Markus Handella3765182020-07-08 11:13:32650 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 10:28:07651 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 12:31:23652 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-03 06:53:04653};
654
Markus Handellb4e96d42021-11-05 11:00:55655class SimpleVideoStreamEncoderFactory {
656 public:
657 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
658 public:
659 using VideoStreamEncoder::VideoStreamEncoder;
660 ~AdaptedVideoStreamEncoder() { Stop(); }
661 };
662
Markus Handell8d87c462021-12-16 10:37:16663 class MockFakeEncoder : public test::FakeEncoder {
664 public:
665 using FakeEncoder::FakeEncoder;
666 MOCK_METHOD(CodecSpecificInfo,
667 EncodeHook,
668 (EncodedImage & encoded_image,
669 rtc::scoped_refptr<EncodedImageBuffer> buffer),
670 (override));
671 };
672
Markus Handellee225432021-11-29 11:35:12673 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 11:00:55674 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 11:35:12675 encoder_settings_.bitrate_allocator_factory =
676 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 11:00:55677 }
678
Markus Handell818e7fb2021-12-30 12:01:33679 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 11:35:12680 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Markus Handell818e7fb2021-12-30 12:01:33681 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue) {
Markus Handellb4e96d42021-11-05 11:00:55682 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
683 time_controller_.GetClock(),
684 /*number_of_cores=*/1,
685 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
686 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 11:35:12687 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 11:00:55688 VideoStreamEncoder::BitrateAllocationCallbackType::
689 kVideoBitrateAllocation);
690 result->SetSink(&sink_, /*rotation_applied=*/false);
691 return result;
692 }
693
Markus Handell818e7fb2021-12-30 12:01:33694 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
695 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
696 TaskQueueBase** encoder_queue_ptr = nullptr) {
697 auto encoder_queue =
698 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
699 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
700 if (encoder_queue_ptr)
701 *encoder_queue_ptr = encoder_queue.get();
702 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
703 std::move(encoder_queue));
704 }
705
Markus Handell9a478b52021-11-18 15:07:01706 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 10:37:16707 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 15:07:01708
Markus Handell818e7fb2021-12-30 12:01:33709 GlobalSimulatedTimeController* GetTimeController() {
710 return &time_controller_;
711 }
712
Markus Handellb4e96d42021-11-05 11:00:55713 private:
714 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
715 public:
716 ~NullEncoderSink() override = default;
717 void OnEncoderConfigurationChanged(
718 std::vector<VideoStream> streams,
719 bool is_svc,
720 VideoEncoderConfig::ContentType content_type,
721 int min_transmit_bitrate_bps) override {}
722 void OnBitrateAllocationUpdated(
723 const VideoBitrateAllocation& allocation) override {}
724 void OnVideoLayersAllocationUpdated(
725 VideoLayersAllocation allocation) override {}
726 Result OnEncodedImage(
727 const EncodedImage& encoded_image,
728 const CodecSpecificInfo* codec_specific_info) override {
729 return Result(EncodedImageCallback::Result::OK);
730 }
731 };
732
Markus Handellee225432021-11-29 11:35:12733 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
734 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
735 time_controller_.CreateTaskQueueFactory()};
736 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
737 std::make_unique<MockableSendStatisticsProxy>(
738 time_controller_.GetClock(),
739 VideoSendStream::Config(nullptr),
740 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
741 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
742 CreateBuiltinVideoBitrateAllocatorFactory();
743 VideoStreamEncoderSettings encoder_settings_{
744 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 10:37:16745 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
746 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 11:00:55747 NullEncoderSink sink_;
748};
749
750class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
751 public:
752 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 10:37:16753 MOCK_METHOD(void,
754 SetZeroHertzModeEnabled,
755 (absl::optional<ZeroHertzModeParams>),
756 (override));
Markus Handellb4e96d42021-11-05 11:00:55757 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 11:35:12758 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
759 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 10:37:16760 MOCK_METHOD(void,
761 UpdateLayerQualityConvergence,
762 (int spatial_index, bool converged),
763 (override));
764 MOCK_METHOD(void,
765 UpdateLayerStatus,
766 (int spatial_index, bool enabled),
767 (override));
Markus Handell818e7fb2021-12-30 12:01:33768 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 11:00:55769};
770
philipel9b058032020-02-10 10:30:00771class MockEncoderSelector
772 : public VideoEncoderFactory::EncoderSelectorInterface {
773 public:
Danil Chapovalov91fdc602020-05-14 17:17:51774 MOCK_METHOD(void,
775 OnCurrentEncoder,
776 (const SdpVideoFormat& format),
777 (override));
778 MOCK_METHOD(absl::optional<SdpVideoFormat>,
779 OnAvailableBitrate,
780 (const DataRate& rate),
781 (override));
782 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 10:30:00783};
784
Markus Handell2e0f4f02021-12-21 18:14:58785class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
786 public:
787 MOCK_METHOD(void,
788 AddOrUpdateSink,
789 (rtc::VideoSinkInterface<VideoFrame>*,
790 const rtc::VideoSinkWants&),
791 (override));
792 MOCK_METHOD(void,
793 RemoveSink,
794 (rtc::VideoSinkInterface<VideoFrame>*),
795 (override));
796 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
797};
798
perkj803d97f2016-11-01 18:45:46799} // namespace
800
mflodmancc3d4422017-08-03 15:27:51801class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 08:17:40802 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23803 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 08:17:40804
mflodmancc3d4422017-08-03 15:27:51805 VideoStreamEncoderTest()
perkj26091b12016-09-01 08:17:40806 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-03 06:45:26807 codec_width_(320),
808 codec_height_(240),
Åsa Persson8c1bf952018-09-13 08:42:19809 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 12:31:23810 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 07:04:13811 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-03 06:53:04812 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 12:31:23813 time_controller_.GetClock(),
perkj803d97f2016-11-01 18:45:46814 video_send_config_,
815 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 12:31:23816 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 08:17:40817
818 void SetUp() override {
perkj803d97f2016-11-01 18:45:46819 metrics::Reset();
perkj26091b12016-09-01 08:17:40820 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 07:04:13821 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56822 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 12:18:34823 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 13:36:51824 video_send_config_.rtp.payload_name = "FAKE";
825 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 08:17:40826
Per512ecb32016-09-23 13:52:06827 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51828 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 06:57:51829 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
830 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
831 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 15:41:30832 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 14:06:52833
Niels Möllerf1338562018-04-26 07:51:47834 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 12:37:00835 }
836
Per Kjellanderb03b6c82021-01-03 09:26:03837 void ConfigureEncoder(
838 VideoEncoderConfig video_encoder_config,
839 VideoStreamEncoder::BitrateAllocationCallbackType
840 allocation_callback_type =
841 VideoStreamEncoder::BitrateAllocationCallbackType::
842 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 15:27:51843 if (video_stream_encoder_)
844 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 15:07:01845
846 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
847 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
848 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
849 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
850 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
851 encoder_queue_ptr);
852 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
853 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
854 stats_proxy_.get(), video_send_config_.encoder_settings,
855 allocation_callback_type);
Asa Persson606d3cb2021-10-04 08:07:11856 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 15:27:51857 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:41858 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 08:07:11859 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 15:27:51860 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:47861 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:51862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 12:37:00863 }
864
Per Kjellanderb03b6c82021-01-03 09:26:03865 void ResetEncoder(const std::string& payload_name,
866 size_t num_streams,
867 size_t num_temporal_layers,
868 unsigned char num_spatial_layers,
869 bool screenshare,
870 VideoStreamEncoder::BitrateAllocationCallbackType
871 allocation_callback_type =
872 VideoStreamEncoder::BitrateAllocationCallbackType::
873 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 13:36:51874 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 12:37:00875
876 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:51877 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
878 num_streams, &video_encoder_config);
879 for (auto& layer : video_encoder_config.simulcast_layers) {
880 layer.num_temporal_layers = num_temporal_layers;
881 layer.max_framerate = kDefaultFramerate;
882 }
Erik Språngd7329ca2019-02-21 20:19:53883 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 08:07:11884 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 14:06:52885 video_encoder_config.content_type =
886 screenshare ? VideoEncoderConfig::ContentType::kScreen
887 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 07:28:40888 if (payload_name == "VP9") {
889 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
890 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 13:29:23891 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 07:28:40892 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43893 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
894 vp9_settings);
emircanbbcc3562017-08-18 07:28:40895 }
Per Kjellanderb03b6c82021-01-03 09:26:03896 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 08:17:40897 }
898
sprang57c2fff2017-01-16 14:24:02899 VideoFrame CreateFrame(int64_t ntp_time_ms,
900 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 17:05:30901 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43902 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 17:05:30903 destruction_event, codec_width_, codec_height_))
904 .set_ntp_time_ms(ntp_time_ms)
905 .set_timestamp_ms(99)
906 .set_rotation(kVideoRotation_0)
907 .build();
perkj26091b12016-09-01 08:17:40908 }
909
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26910 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
911 rtc::Event* destruction_event,
912 int offset_x) const {
Åsa Persson90719572021-04-08 17:05:30913 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43914 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 17:05:30915 destruction_event, codec_width_, codec_height_))
916 .set_ntp_time_ms(ntp_time_ms)
917 .set_timestamp_ms(99)
918 .set_rotation(kVideoRotation_0)
919 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
920 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26921 }
922
sprang57c2fff2017-01-16 14:24:02923 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 12:02:13924 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
925 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 17:05:30926 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 12:02:13927 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 17:05:30928 .set_ntp_time_ms(ntp_time_ms)
929 .set_timestamp_ms(ntp_time_ms)
930 .set_rotation(kVideoRotation_0)
931 .build();
perkj803d97f2016-11-01 18:45:46932 }
933
Evan Shrubsole895556e2020-10-05 07:15:13934 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 17:05:30935 return VideoFrame::Builder()
936 .set_video_frame_buffer(NV12Buffer::Create(width, height))
937 .set_ntp_time_ms(ntp_time_ms)
938 .set_timestamp_ms(ntp_time_ms)
939 .set_rotation(kVideoRotation_0)
940 .build();
Evan Shrubsole895556e2020-10-05 07:15:13941 }
942
Noah Richards51db4212019-06-12 13:59:12943 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
944 rtc::Event* destruction_event,
945 int width,
946 int height) const {
Åsa Persson90719572021-04-08 17:05:30947 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43948 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 17:05:30949 destruction_event, width, height))
950 .set_ntp_time_ms(ntp_time_ms)
951 .set_timestamp_ms(99)
952 .set_rotation(kVideoRotation_0)
953 .build();
Noah Richards51db4212019-06-12 13:59:12954 }
955
Evan Shrubsole895556e2020-10-05 07:15:13956 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
957 rtc::Event* destruction_event,
958 int width,
959 int height) const {
Åsa Persson90719572021-04-08 17:05:30960 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43961 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 17:05:30962 destruction_event, width, height))
963 .set_ntp_time_ms(ntp_time_ms)
964 .set_timestamp_ms(99)
965 .set_rotation(kVideoRotation_0)
966 .build();
Evan Shrubsole895556e2020-10-05 07:15:13967 }
968
Noah Richards51db4212019-06-12 13:59:12969 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
970 rtc::Event* destruction_event) const {
971 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
972 codec_height_);
973 }
974
Åsa Perssonc29cb2c2019-03-25 11:06:59975 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 16:49:07976 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:11977 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 11:06:59978
979 video_source_.IncomingCapturedFrame(
980 CreateFrame(1, codec_width_, codec_height_));
981 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 13:09:05982 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 11:06:59983 }
984
sprang4847ae62017-06-27 14:06:52985 void WaitForEncodedFrame(int64_t expected_ntp_time) {
986 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 12:31:23987 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52988 }
989
990 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
991 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:23992 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52993 return ok;
994 }
995
996 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
997 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 12:31:23998 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52999 }
1000
1001 void ExpectDroppedFrame() {
1002 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 12:31:231003 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:521004 }
1005
1006 bool WaitForFrame(int64_t timeout_ms) {
1007 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:231008 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:521009 return ok;
1010 }
1011
perkj26091b12016-09-01 08:17:401012 class TestEncoder : public test::FakeEncoder {
1013 public:
Tomas Gunnarsson612445e2020-09-21 12:31:231014 explicit TestEncoder(TimeController* time_controller)
1015 : FakeEncoder(time_controller->GetClock()),
1016 time_controller_(time_controller) {
1017 RTC_DCHECK(time_controller_);
1018 }
perkj26091b12016-09-01 08:17:401019
Erik Språngaed30702018-11-05 11:57:171020 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 11:13:321021 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 15:44:421022 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 14:52:331023 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 11:42:181024 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 14:56:001025 info.scaling_settings = VideoEncoder::ScalingSettings(
1026 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 11:42:181027 }
1028 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 11:06:591029 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1030 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 14:27:351031 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 11:06:591032 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 14:27:351033 for (int tid = 0; tid < num_layers; ++tid)
1034 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 11:06:591035 }
1036 }
Erik Språngaed30702018-11-05 11:57:171037 }
Sergey Silkin6456e352019-07-08 15:56:401038
1039 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 08:47:111040 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 15:50:001041 info.apply_alignment_to_all_simulcast_layers =
1042 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 12:56:451043 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-29 22:48:031044 if (is_qp_trusted_.has_value()) {
1045 info.is_qp_trusted = is_qp_trusted_;
1046 }
Erik Språngaed30702018-11-05 11:57:171047 return info;
kthelgason876222f2016-11-29 09:44:111048 }
1049
Erik Språngb7cb7b52019-02-26 14:52:331050 int32_t RegisterEncodeCompleteCallback(
1051 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 11:13:321052 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331053 encoded_image_callback_ = callback;
1054 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1055 }
1056
perkjfa10b552016-10-03 06:45:261057 void ContinueEncode() { continue_encode_event_.Set(); }
1058
1059 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1060 uint32_t timestamp) const {
Markus Handella3765182020-07-08 11:13:321061 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-03 06:45:261062 EXPECT_EQ(timestamp_, timestamp);
1063 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1064 }
1065
kthelgason2fc52542017-03-03 08:24:411066 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 11:13:321067 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 08:24:411068 quality_scaling_ = b;
1069 }
kthelgasonad9010c2017-02-14 08:46:511070
Rasmus Brandt5cad55b2019-12-19 08:47:111071 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 11:13:321072 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 08:47:111073 requested_resolution_alignment_ = requested_resolution_alignment;
1074 }
1075
Åsa Perssonc5a74ff2020-09-20 15:50:001076 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1077 MutexLock lock(&local_mutex_);
1078 apply_alignment_to_all_simulcast_layers_ = b;
1079 }
1080
Mirta Dvornicicccc1b572019-01-15 11:42:181081 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 11:13:321082 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 11:42:181083 is_hardware_accelerated_ = is_hardware_accelerated;
1084 }
1085
Åsa Perssonc29cb2c2019-03-25 11:06:591086 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1087 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 11:13:321088 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:591089 temporal_layers_supported_[spatial_idx] = supported;
1090 }
1091
Sergey Silkin6456e352019-07-08 15:56:401092 void SetResolutionBitrateLimits(
1093 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 11:13:321094 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 15:56:401095 resolution_bitrate_limits_ = thresholds;
1096 }
1097
sprangfe627f32017-03-29 15:24:591098 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 11:13:321099 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 15:24:591100 force_init_encode_failed_ = force_failure;
1101 }
1102
Niels Möller6bb5ab92019-01-11 10:11:101103 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 11:13:321104 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:101105 rate_factor_ = rate_factor;
1106 }
1107
Erik Språngd7329ca2019-02-21 20:19:531108 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 11:13:321109 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:101110 return last_framerate_;
1111 }
1112
Erik Språngd7329ca2019-02-21 20:19:531113 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 11:13:321114 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:261115 return last_update_rect_;
1116 }
1117
Niels Möller87e2d782019-03-07 09:18:231118 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 11:13:321119 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531120 return last_frame_types_;
1121 }
1122
1123 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 09:18:231124 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 14:43:581125 keyframe ? VideoFrameType::kVideoFrameKey
1126 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:531127 {
Markus Handella3765182020-07-08 11:13:321128 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531129 last_frame_types_ = frame_type;
1130 }
Niels Möllerb859b322019-03-07 11:40:011131 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 20:19:531132 }
1133
Sergey Silkin0e42cf72021-03-15 09:12:571134 void InjectEncodedImage(const EncodedImage& image,
1135 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 11:13:321136 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 09:12:571137 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 14:52:331138 }
1139
Mirta Dvornicic97910da2020-07-14 13:29:231140 void SetEncodedImageData(
1141 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 11:13:321142 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 13:29:231143 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 14:30:161144 }
1145
Erik Språngd7329ca2019-02-21 20:19:531146 void ExpectNullFrame() {
Markus Handella3765182020-07-08 11:13:321147 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531148 expect_null_frame_ = true;
1149 }
1150
Erik Språng5056af02019-09-02 13:53:111151 absl::optional<VideoEncoder::RateControlParameters>
1152 GetAndResetLastRateControlSettings() {
1153 auto settings = last_rate_control_settings_;
1154 last_rate_control_settings_.reset();
1155 return settings;
Erik Språngd7329ca2019-02-21 20:19:531156 }
1157
Henrik Boström56db9ff2021-03-24 08:06:451158 int GetLastInputWidth() const {
1159 MutexLock lock(&local_mutex_);
1160 return last_input_width_;
1161 }
1162
1163 int GetLastInputHeight() const {
1164 MutexLock lock(&local_mutex_);
1165 return last_input_height_;
1166 }
1167
Evan Shrubsole895556e2020-10-05 07:15:131168 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1169 MutexLock lock(&local_mutex_);
1170 return last_input_pixel_format_;
1171 }
1172
Evan Shrubsole7c079f62019-09-26 07:55:031173 int GetNumSetRates() const {
Markus Handella3765182020-07-08 11:13:321174 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031175 return num_set_rates_;
1176 }
1177
Evan Shrubsoleb556b082020-10-08 12:56:451178 void SetPreferredPixelFormats(
1179 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1180 pixel_formats) {
1181 MutexLock lock(&local_mutex_);
1182 preferred_pixel_formats_ = std::move(pixel_formats);
1183 }
1184
Qiu Jianlinb54cfde2021-07-29 22:48:031185 void SetIsQpTrusted(absl::optional<bool> trusted) {
1186 MutexLock lock(&local_mutex_);
1187 is_qp_trusted_ = trusted;
1188 }
1189
perkjfa10b552016-10-03 06:45:261190 private:
perkj26091b12016-09-01 08:17:401191 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 09:18:231192 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 08:17:401193 {
Markus Handella3765182020-07-08 11:13:321194 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531195 if (expect_null_frame_) {
1196 EXPECT_EQ(input_image.timestamp(), 0u);
1197 EXPECT_EQ(input_image.width(), 1);
1198 last_frame_types_ = *frame_types;
1199 expect_null_frame_ = false;
1200 } else {
1201 EXPECT_GT(input_image.timestamp(), timestamp_);
1202 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1203 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1204 }
perkj26091b12016-09-01 08:17:401205
1206 timestamp_ = input_image.timestamp();
1207 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 18:45:461208 last_input_width_ = input_image.width();
1209 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:261210 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 20:19:531211 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 07:15:131212 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 08:17:401213 }
Niels Möllerb859b322019-03-07 11:40:011214 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 08:17:401215 return result;
1216 }
1217
Niels Möller08ae7ce2020-09-23 13:58:121218 CodecSpecificInfo EncodeHook(
1219 EncodedImage& encoded_image,
1220 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 15:30:361221 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231222 {
1223 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 15:30:361224 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 13:29:231225 }
1226 MutexLock lock(&local_mutex_);
1227 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 15:30:361228 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 13:29:231229 }
Danil Chapovalov2549f172020-08-12 15:30:361230 return codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231231 }
1232
sprangfe627f32017-03-29 15:24:591233 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 12:57:571234 const Settings& settings) override {
1235 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 12:18:341236
Markus Handella3765182020-07-08 11:13:321237 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331238 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 12:18:341239
Erik Språng82fad3d2018-03-21 08:57:231240 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 15:24:591241 // Simulate setting up temporal layers, in order to validate the life
1242 // cycle of these objects.
Elad Aloncde8ab22019-03-20 10:56:201243 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 09:20:091244 frame_buffer_controller_ =
1245 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 15:24:591246 }
Erik Språngb7cb7b52019-02-26 14:52:331247 if (force_init_encode_failed_) {
1248 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 15:24:591249 return -1;
Erik Språngb7cb7b52019-02-26 14:52:331250 }
Mirta Dvornicicccc1b572019-01-15 11:42:181251
Erik Språngb7cb7b52019-02-26 14:52:331252 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 15:24:591253 return res;
1254 }
1255
Erik Språngb7cb7b52019-02-26 14:52:331256 int32_t Release() override {
Markus Handella3765182020-07-08 11:13:321257 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331258 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1259 initialized_ = EncoderState::kUninitialized;
1260 return FakeEncoder::Release();
1261 }
1262
Erik Språng16cb8f52019-04-12 11:59:091263 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 11:13:321264 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031265 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 10:11:101266 VideoBitrateAllocation adjusted_rate_allocation;
1267 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1268 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 11:59:091269 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 10:11:101270 adjusted_rate_allocation.SetBitrate(
1271 si, ti,
Erik Språng16cb8f52019-04-12 11:59:091272 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 10:11:101273 rate_factor_));
1274 }
1275 }
1276 }
Erik Språng16cb8f52019-04-12 11:59:091277 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 13:53:111278 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 11:59:091279 RateControlParameters adjusted_paramters = parameters;
1280 adjusted_paramters.bitrate = adjusted_rate_allocation;
1281 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 10:11:101282 }
1283
Tomas Gunnarsson612445e2020-09-21 12:31:231284 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321285 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 14:52:331286 enum class EncoderState {
1287 kUninitialized,
1288 kInitializationFailed,
1289 kInitialized
Markus Handella3765182020-07-08 11:13:321290 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 08:17:401291 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 11:13:321292 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1293 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1294 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1295 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1296 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1297 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 15:50:001298 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1299 false;
Markus Handella3765182020-07-08 11:13:321300 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 13:29:231301 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1302 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 10:56:201303 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 11:13:321304 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:591305 absl::optional<bool>
1306 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 11:13:321307 local_mutex_);
1308 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1309 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1310 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 13:53:111311 absl::optional<VideoEncoder::RateControlParameters>
1312 last_rate_control_settings_;
Markus Handella3765182020-07-08 11:13:321313 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1314 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 09:18:231315 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 20:19:531316 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 11:13:321317 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1318 nullptr;
Evan Shrubsolefb862742020-03-16 15:18:361319 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 15:56:401320 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 11:13:321321 RTC_GUARDED_BY(local_mutex_);
1322 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 07:15:131323 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1324 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 12:56:451325 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1326 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-29 22:48:031327 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 08:17:401328 };
1329
mflodmancc3d4422017-08-03 15:27:511330 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 08:17:401331 public:
Tomas Gunnarsson612445e2020-09-21 12:31:231332 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1333 : time_controller_(time_controller), test_encoder_(test_encoder) {
1334 RTC_DCHECK(time_controller_);
1335 }
perkj26091b12016-09-01 08:17:401336
perkj26091b12016-09-01 08:17:401337 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 14:06:521338 EXPECT_TRUE(
1339 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1340 }
1341
1342 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1343 int64_t timeout_ms) {
perkj26091b12016-09-01 08:17:401344 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:231345 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 14:06:521346 return false;
perkj26091b12016-09-01 08:17:401347 {
Markus Handella3765182020-07-08 11:13:321348 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121349 timestamp = last_timestamp_;
perkj26091b12016-09-01 08:17:401350 }
1351 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 14:06:521352 return true;
perkj26091b12016-09-01 08:17:401353 }
1354
sprangb1ca0732017-02-01 16:38:121355 void WaitForEncodedFrame(uint32_t expected_width,
1356 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 12:31:231357 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 13:13:561358 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-03 06:53:041359 }
1360
Åsa Perssonc74d8da2017-12-04 13:13:561361 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-03 06:53:041362 uint32_t expected_height) {
sprangb1ca0732017-02-01 16:38:121363 uint32_t width = 0;
1364 uint32_t height = 0;
sprangb1ca0732017-02-01 16:38:121365 {
Markus Handella3765182020-07-08 11:13:321366 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121367 width = last_width_;
1368 height = last_height_;
1369 }
1370 EXPECT_EQ(expected_height, height);
1371 EXPECT_EQ(expected_width, width);
1372 }
1373
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141374 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1375 VideoRotation rotation;
1376 {
Markus Handella3765182020-07-08 11:13:321377 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141378 rotation = last_rotation_;
1379 }
1380 EXPECT_EQ(expected_rotation, rotation);
1381 }
1382
Tomas Gunnarsson612445e2020-09-21 12:31:231383 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 15:02:221384
sprangc5d62e22017-04-03 06:53:041385 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 12:31:231386 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 09:11:551387 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:231388 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 09:11:551389 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 12:31:231390 return ret;
sprangc5d62e22017-04-03 06:53:041391 }
1392
perkj26091b12016-09-01 08:17:401393 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 11:13:321394 MutexLock lock(&mutex_);
perkj26091b12016-09-01 08:17:401395 expect_frames_ = false;
1396 }
1397
asaperssonfab67072017-04-04 12:51:491398 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 11:13:321399 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061400 return number_of_reconfigurations_;
1401 }
1402
asaperssonfab67072017-04-04 12:51:491403 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 11:13:321404 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061405 return min_transmit_bitrate_bps_;
1406 }
1407
Erik Språngd7329ca2019-02-21 20:19:531408 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 11:13:321409 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 20:19:531410 num_expected_layers_ = num_layers;
1411 }
1412
Erik Språngb7cb7b52019-02-26 14:52:331413 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 11:13:321414 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331415 return last_capture_time_ms_;
1416 }
1417
Sergey Silkin0e42cf72021-03-15 09:12:571418 const EncodedImage& GetLastEncodedImage() {
1419 MutexLock lock(&mutex_);
1420 return last_encoded_image_;
1421 }
1422
Mirta Dvornicic28f0eb22019-05-28 14:30:161423 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 11:13:321424 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 14:30:161425 return std::move(last_encoded_image_data_);
1426 }
1427
Per Kjellanderdcef6412020-10-07 13:09:051428 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1429 MutexLock lock(&mutex_);
1430 return last_bitrate_allocation_;
1431 }
1432
1433 int number_of_bitrate_allocations() const {
1434 MutexLock lock(&mutex_);
1435 return number_of_bitrate_allocations_;
1436 }
1437
Per Kjellandera9434842020-10-15 15:53:221438 VideoLayersAllocation GetLastVideoLayersAllocation() {
1439 MutexLock lock(&mutex_);
1440 return last_layers_allocation_;
1441 }
1442
1443 int number_of_layers_allocations() const {
1444 MutexLock lock(&mutex_);
1445 return number_of_layers_allocations_;
1446 }
1447
perkj26091b12016-09-01 08:17:401448 private:
sergeyu2cb155a2016-11-04 18:39:291449 Result OnEncodedImage(
1450 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 15:30:361451 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 11:13:321452 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061453 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 09:12:571454 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 14:30:161455 last_encoded_image_data_ = std::vector<uint8_t>(
1456 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 20:19:531457 uint32_t timestamp = encoded_image.Timestamp();
1458 if (last_timestamp_ != timestamp) {
1459 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 12:02:131460 last_width_ = encoded_image._encodedWidth;
1461 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 20:19:531462 } else {
1463 ++num_received_layers_;
Erik Språng7444b192021-06-02 12:02:131464 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1465 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 20:19:531466 }
1467 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 14:52:331468 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141469 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 20:19:531470 if (num_received_layers_ == num_expected_layers_) {
1471 encoded_frame_event_.Set();
1472 }
sprangb1ca0732017-02-01 16:38:121473 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 13:52:061474 }
1475
Rasmus Brandtc402dbe2019-02-04 10:09:461476 void OnEncoderConfigurationChanged(
1477 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 12:10:271478 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 10:09:461479 VideoEncoderConfig::ContentType content_type,
1480 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 11:13:321481 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061482 ++number_of_reconfigurations_;
1483 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1484 }
1485
Per Kjellanderdcef6412020-10-07 13:09:051486 void OnBitrateAllocationUpdated(
1487 const VideoBitrateAllocation& allocation) override {
1488 MutexLock lock(&mutex_);
1489 ++number_of_bitrate_allocations_;
1490 last_bitrate_allocation_ = allocation;
1491 }
1492
Per Kjellandera9434842020-10-15 15:53:221493 void OnVideoLayersAllocationUpdated(
1494 VideoLayersAllocation allocation) override {
1495 MutexLock lock(&mutex_);
1496 ++number_of_layers_allocations_;
1497 last_layers_allocation_ = allocation;
1498 rtc::StringBuilder log;
1499 for (const auto& layer : allocation.active_spatial_layers) {
1500 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1501 << "[";
1502 for (const auto target_bitrate :
1503 layer.target_bitrate_per_temporal_layer) {
1504 log << target_bitrate.kbps() << ",";
1505 }
1506 log << "]";
1507 }
Harald Alvestrand97597c02021-11-04 12:01:231508 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 15:53:221509 }
1510
Tomas Gunnarsson612445e2020-09-21 12:31:231511 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321512 mutable Mutex mutex_;
perkj26091b12016-09-01 08:17:401513 TestEncoder* test_encoder_;
1514 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 09:12:571515 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 14:30:161516 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 16:38:121517 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 14:52:331518 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 16:38:121519 uint32_t last_height_ = 0;
1520 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141521 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 20:19:531522 size_t num_expected_layers_ = 1;
1523 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 08:17:401524 bool expect_frames_ = true;
Per512ecb32016-09-23 13:52:061525 int number_of_reconfigurations_ = 0;
1526 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 13:09:051527 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1528 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 15:53:221529 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1530 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 08:17:401531 };
1532
Sergey Silkin5ee69672019-07-02 12:18:341533 class VideoBitrateAllocatorProxyFactory
1534 : public VideoBitrateAllocatorFactory {
1535 public:
1536 VideoBitrateAllocatorProxyFactory()
1537 : bitrate_allocator_factory_(
1538 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1539
1540 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1541 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 11:13:321542 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341543 codec_config_ = codec;
1544 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1545 }
1546
1547 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 11:13:321548 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341549 return codec_config_;
1550 }
1551
1552 private:
1553 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1554
Markus Handella3765182020-07-08 11:13:321555 mutable Mutex mutex_;
1556 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341557 };
1558
Tomas Gunnarsson612445e2020-09-21 12:31:231559 Clock* clock() { return time_controller_.GetClock(); }
1560 void AdvanceTime(TimeDelta duration) {
1561 time_controller_.AdvanceTime(duration);
1562 }
1563
1564 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1565
1566 protected:
1567 virtual TaskQueueFactory* GetTaskQueueFactory() {
1568 return time_controller_.GetTaskQueueFactory();
1569 }
1570
1571 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 08:17:401572 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 15:41:301573 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 13:52:061574 int codec_width_;
1575 int codec_height_;
sprang4847ae62017-06-27 14:06:521576 int max_framerate_;
perkj26091b12016-09-01 08:17:401577 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 07:07:241578 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 12:18:341579 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-03 06:53:041580 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 08:17:401581 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 12:31:231582 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 15:27:511583 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 08:17:401584};
1585
mflodmancc3d4422017-08-03 15:27:511586TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 16:49:071587 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111588 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 07:43:501589 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411590 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 14:06:521591 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 14:53:411592 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 15:27:511593 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401594}
1595
mflodmancc3d4422017-08-03 15:27:511596TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 08:17:401597 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 07:43:501598 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 11:05:491599 // The encoder will cache up to one frame for a short duration. Adding two
1600 // frames means that the first frame will be dropped and the second frame will
1601 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 14:53:411602 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 12:31:231603 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 11:05:491604 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 09:11:551605 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 14:53:411606 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401607
Henrik Boström381d1092020-05-12 16:49:071608 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111609 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401610
Sebastian Janssona3177052018-04-10 11:05:491611 // The pending frame should be received.
sprang4847ae62017-06-27 14:06:521612 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 11:05:491613 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1614
1615 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:511616 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401617}
1618
mflodmancc3d4422017-08-03 15:27:511619TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 16:49:071620 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111621 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 14:53:411622 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521623 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401624
Henrik Boström381d1092020-05-12 16:49:071625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111626 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1627
Sebastian Janssona3177052018-04-10 11:05:491628 // The encoder will cache up to one frame for a short duration. Adding two
1629 // frames means that the first frame will be dropped and the second frame will
1630 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 14:53:411631 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 11:05:491632 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401633
Henrik Boström381d1092020-05-12 16:49:071634 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111635 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:521636 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 11:05:491637 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1638 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 15:27:511639 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401640}
1641
mflodmancc3d4422017-08-03 15:27:511642TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 16:49:071643 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111644 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 14:53:411645 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521646 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401647
1648 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 14:53:411649 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 08:17:401650
perkja49cbd32016-09-16 14:53:411651 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521652 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511653 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401654}
1655
mflodmancc3d4422017-08-03 15:27:511656TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 16:49:071657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111658 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401659
perkja49cbd32016-09-16 14:53:411660 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521661 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401662
mflodmancc3d4422017-08-03 15:27:511663 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401664 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 07:43:501665 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411666 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1667 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401668}
1669
Markus Handell9a478b52021-11-18 15:07:011670TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1671 test::FrameForwarder source;
1672 video_stream_encoder_->SetSource(&source,
1673 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 16:49:071674 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111675 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 08:17:401676
Tomas Gunnarsson612445e2020-09-21 12:31:231677 int dropped_count = 0;
1678 stats_proxy_->SetDroppedFrameCallback(
1679 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1680 ++dropped_count;
1681 });
1682
Markus Handell9a478b52021-11-18 15:07:011683 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1684 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1685 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511686 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 12:31:231687 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 08:17:401688}
1689
Henrik Boström56db9ff2021-03-24 08:06:451690TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 16:49:071691 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111692 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121693
1694 rtc::Event frame_destroyed_event;
1695 video_source_.IncomingCapturedFrame(
1696 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 08:06:451697 WaitForEncodedFrame(1);
1698 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1699 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 08:07:111700 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1701 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 13:59:121702 video_stream_encoder_->Stop();
1703}
1704
Henrik Boström56db9ff2021-03-24 08:06:451705TEST_F(VideoStreamEncoderTest,
1706 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 13:59:121707 // Use the cropping factory.
1708 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:431709 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 13:59:121710 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1711 kMaxPayloadLength);
1712 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1713
1714 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 16:49:071715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111716 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121717 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1718 WaitForEncodedFrame(1);
1719 // The encoder will have been configured once.
1720 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111721 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1722 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 13:59:121723
1724 // Now send in a fake frame that needs to be cropped as the width/height
1725 // aren't divisible by 4 (see CreateEncoderStreams above).
1726 rtc::Event frame_destroyed_event;
1727 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1728 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 08:06:451729 WaitForEncodedFrame(2);
1730 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1731 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 08:07:111732 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1733 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 13:59:121734 video_stream_encoder_->Stop();
1735}
1736
Evan Shrubsole895556e2020-10-05 07:15:131737TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1738 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111739 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 07:15:131740
1741 video_source_.IncomingCapturedFrame(
1742 CreateNV12Frame(1, codec_width_, codec_height_));
1743 WaitForEncodedFrame(1);
1744 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1745 fake_encoder_.GetLastInputPixelFormat());
1746 video_stream_encoder_->Stop();
1747}
1748
Henrik Boström56db9ff2021-03-24 08:06:451749TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 12:56:451750 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111751 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451752
1753 fake_encoder_.SetPreferredPixelFormats({});
1754
1755 rtc::Event frame_destroyed_event;
1756 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1757 1, &frame_destroyed_event, codec_width_, codec_height_));
1758 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451759 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451760 fake_encoder_.GetLastInputPixelFormat());
1761 video_stream_encoder_->Stop();
1762}
1763
Henrik Boström56db9ff2021-03-24 08:06:451764TEST_F(VideoStreamEncoderTest,
1765 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 12:56:451766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451768
1769 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1770
1771 rtc::Event frame_destroyed_event;
1772 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1773 1, &frame_destroyed_event, codec_width_, codec_height_));
1774 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451775 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451776 fake_encoder_.GetLastInputPixelFormat());
1777 video_stream_encoder_->Stop();
1778}
1779
Henrik Boström56db9ff2021-03-24 08:06:451780TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 12:56:451781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111782 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 12:56:451783
1784 // Fake NV12 native frame does not allow mapping to I444.
1785 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1786
1787 rtc::Event frame_destroyed_event;
1788 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1789 1, &frame_destroyed_event, codec_width_, codec_height_));
1790 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451791 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451792 fake_encoder_.GetLastInputPixelFormat());
1793 video_stream_encoder_->Stop();
1794}
1795
Henrik Boström56db9ff2021-03-24 08:06:451796TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 07:15:131797 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111798 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 07:15:131799
1800 rtc::Event frame_destroyed_event;
1801 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1802 1, &frame_destroyed_event, codec_width_, codec_height_));
1803 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451804 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 07:15:131805 fake_encoder_.GetLastInputPixelFormat());
1806 video_stream_encoder_->Stop();
1807}
1808
Ying Wang9b881ab2020-02-07 13:29:321809TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 16:49:071810 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111811 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 13:29:321812 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1813 WaitForEncodedFrame(1);
1814
Henrik Boström381d1092020-05-12 16:49:071815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111816 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 13:29:321817 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1818 // frames. Adding two frames means that the first frame will be dropped and
1819 // the second frame will be sent to the encoder.
1820 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1821 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1822 WaitForEncodedFrame(3);
1823 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1824 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1825 WaitForEncodedFrame(5);
1826 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1827 video_stream_encoder_->Stop();
1828}
1829
mflodmancc3d4422017-08-03 15:27:511830TEST_F(VideoStreamEncoderTest,
1831 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 16:49:071832 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111833 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 20:37:571834 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061835
1836 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551837 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521838 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571839 // The encoder will have been configured once when the first frame is
1840 // received.
1841 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061842
1843 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:511844 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 13:52:061845 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 15:27:511846 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:471847 kMaxPayloadLength);
Per512ecb32016-09-23 13:52:061848
1849 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551850 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521851 WaitForEncodedFrame(2);
Per21d45d22016-10-30 20:37:571852 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-30 06:25:401853 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-30 05:39:101854
mflodmancc3d4422017-08-03 15:27:511855 video_stream_encoder_->Stop();
perkj26105b42016-09-30 05:39:101856}
1857
mflodmancc3d4422017-08-03 15:27:511858TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 16:49:071859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111860 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-03 06:45:261861
1862 // Capture a frame and wait for it to synchronize with the encoder thread.
1863 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521864 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571865 // The encoder will have been configured once.
1866 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111867 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1868 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-03 06:45:261869
1870 codec_width_ *= 2;
1871 codec_height_ *= 2;
1872 // Capture a frame with a higher resolution and wait for it to synchronize
1873 // with the encoder thread.
1874 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521875 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 08:07:111876 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1877 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 20:37:571878 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261879
mflodmancc3d4422017-08-03 15:27:511880 video_stream_encoder_->Stop();
perkjfa10b552016-10-03 06:45:261881}
1882
Sergey Silkin443b7ee2019-06-28 10:53:071883TEST_F(VideoStreamEncoderTest,
1884 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 16:49:071885 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111886 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 10:53:071887
1888 // Capture a frame and wait for it to synchronize with the encoder thread.
1889 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1890 WaitForEncodedFrame(1);
1891
1892 VideoEncoderConfig video_encoder_config;
1893 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1894 // Changing the max payload data length recreates encoder.
1895 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1896 kMaxPayloadLength / 2);
1897
1898 // Capture a frame and wait for it to synchronize with the encoder thread.
1899 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1900 WaitForEncodedFrame(2);
1901 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1902
1903 video_stream_encoder_->Stop();
1904}
1905
Sergey Silkin5ee69672019-07-02 12:18:341906TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 16:49:071907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111908 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 12:18:341909
1910 VideoEncoderConfig video_encoder_config;
1911 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:111912 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1913 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 12:18:341914 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1915 kMaxPayloadLength);
1916
1917 // Capture a frame and wait for it to synchronize with the encoder thread.
1918 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1919 WaitForEncodedFrame(1);
1920 // The encoder will have been configured once when the first frame is
1921 // received.
1922 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 08:07:111923 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 12:18:341924 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111925 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 12:18:341926 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1927
Sergey Silkin6456e352019-07-08 15:56:401928 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1929 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 08:07:111930 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1931 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 12:18:341932 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1933 kMaxPayloadLength);
1934
1935 // Capture a frame and wait for it to synchronize with the encoder thread.
1936 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1937 WaitForEncodedFrame(2);
1938 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1939 // Bitrate limits have changed - rate allocator should be reconfigured,
1940 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 08:07:111941 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 12:18:341942 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111943 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 12:18:341944 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 08:07:111945 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 12:18:341946
1947 video_stream_encoder_->Stop();
1948}
1949
Sergey Silkin6456e352019-07-08 15:56:401950TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 13:48:401951 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 16:49:071952 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111953 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401954
Sergey Silkincd02eba2020-01-20 13:48:401955 const uint32_t kMinEncBitrateKbps = 100;
1956 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051957 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 13:48:401958 /*frame_size_pixels=*/codec_width_ * codec_height_,
1959 /*min_start_bitrate_bps=*/0,
1960 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1961 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051962 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1963
Sergey Silkincd02eba2020-01-20 13:48:401964 VideoEncoderConfig video_encoder_config;
1965 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1966 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1967 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1968 (kMinEncBitrateKbps + 1) * 1000;
1969 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1970 kMaxPayloadLength);
1971
1972 // When both encoder and app provide bitrate limits, the intersection of
1973 // provided sets should be used.
1974 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1975 WaitForEncodedFrame(1);
1976 EXPECT_EQ(kMaxEncBitrateKbps,
1977 bitrate_allocator_factory_.codec_config().maxBitrate);
1978 EXPECT_EQ(kMinEncBitrateKbps + 1,
1979 bitrate_allocator_factory_.codec_config().minBitrate);
1980
1981 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1982 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1983 (kMinEncBitrateKbps - 1) * 1000;
1984 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1985 kMaxPayloadLength);
1986 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 14:04:051987 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 13:48:401988 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 14:04:051989 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:401990 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051991 bitrate_allocator_factory_.codec_config().minBitrate);
1992
Sergey Silkincd02eba2020-01-20 13:48:401993 video_stream_encoder_->Stop();
1994}
1995
1996TEST_F(VideoStreamEncoderTest,
1997 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 16:49:071998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:111999 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 13:48:402000
2001 const uint32_t kMinAppBitrateKbps = 100;
2002 const uint32_t kMaxAppBitrateKbps = 200;
2003 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2004 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2005 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2006 /*frame_size_pixels=*/codec_width_ * codec_height_,
2007 /*min_start_bitrate_bps=*/0,
2008 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2009 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2010 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2011
2012 VideoEncoderConfig video_encoder_config;
2013 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2014 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2015 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2016 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 14:04:052017 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2018 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 14:04:052019
Sergey Silkincd02eba2020-01-20 13:48:402020 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2021 WaitForEncodedFrame(1);
2022 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:052023 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:402024 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:052025 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 15:56:402026
2027 video_stream_encoder_->Stop();
2028}
2029
2030TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 14:04:052031 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 16:49:072032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112033 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:402034
2035 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 14:04:052036 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 15:56:402037 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 14:04:052038 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 15:56:402039 fake_encoder_.SetResolutionBitrateLimits(
2040 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2041
2042 VideoEncoderConfig video_encoder_config;
2043 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2044 video_encoder_config.max_bitrate_bps = 0;
2045 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2046 kMaxPayloadLength);
2047
Sergey Silkin6b2cec12019-08-09 14:04:052048 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 15:56:402049 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2050 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 14:04:052051 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2052 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402053 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2054 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2055
Sergey Silkin6b2cec12019-08-09 14:04:052056 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 15:56:402057 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2058 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 14:04:052059 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2060 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402061 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2062 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2063
Sergey Silkin6b2cec12019-08-09 14:04:052064 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 15:56:402065 // encoder for 360p should be used.
2066 video_source_.IncomingCapturedFrame(
2067 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2068 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 14:04:052069 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2070 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402071 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2072 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2073
Sergey Silkin6b2cec12019-08-09 14:04:052074 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 15:56:402075 // ignored.
2076 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2077 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 14:04:052078 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2079 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402080 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2081 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:052082 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2083 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402084 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2085 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2086
2087 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2088 // for 270p should be used.
2089 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2090 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 14:04:052091 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2092 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402093 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2094 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2095
2096 video_stream_encoder_->Stop();
2097}
2098
Sergey Silkin6b2cec12019-08-09 14:04:052099TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 16:49:072100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112101 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 14:04:052102
2103 VideoEncoderConfig video_encoder_config;
2104 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2105 video_encoder_config.max_bitrate_bps = 0;
2106 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2107 kMaxPayloadLength);
2108
2109 // Encode 720p frame to get the default encoder target bitrate.
2110 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2111 WaitForEncodedFrame(1);
2112 const uint32_t kDefaultTargetBitrateFor720pKbps =
2113 bitrate_allocator_factory_.codec_config()
2114 .simulcastStream[0]
2115 .targetBitrate;
2116
2117 // Set the max recommended encoder bitrate to something lower than the default
2118 // target bitrate.
2119 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2120 1280 * 720, 10 * 1000, 10 * 1000,
2121 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2122 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2123
2124 // Change resolution to trigger encoder reinitialization.
2125 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2126 WaitForEncodedFrame(2);
2127 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2128 WaitForEncodedFrame(3);
2129
2130 // Ensure the target bitrate is capped by the max bitrate.
2131 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2132 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2133 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2134 .simulcastStream[0]
2135 .targetBitrate *
2136 1000,
2137 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2138
2139 video_stream_encoder_->Stop();
2140}
2141
Åsa Perssona7e34d32021-01-20 14:36:132142TEST_F(VideoStreamEncoderTest,
2143 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2144 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2145 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2146 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2147 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2148 fake_encoder_.SetResolutionBitrateLimits(
2149 {kEncoderLimits270p, kEncoderLimits360p});
2150
2151 // Two streams, highest stream active.
2152 VideoEncoderConfig config;
2153 const int kNumStreams = 2;
2154 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2155 config.max_bitrate_bps = 0;
2156 config.simulcast_layers[0].active = false;
2157 config.simulcast_layers[1].active = true;
2158 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432159 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132160 "VP8", /*max qp*/ 56, /*screencast*/ false,
2161 /*screenshare enabled*/ false);
2162 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2163
2164 // The encoder bitrate limits for 270p should be used.
2165 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2166 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112167 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132168 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112169 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132170 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112171 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132172
2173 // The encoder bitrate limits for 360p should be used.
2174 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2175 EXPECT_FALSE(WaitForFrame(1000));
2176 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112177 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132178 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112179 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132180
2181 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2182 video_source_.IncomingCapturedFrame(
2183 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2184 EXPECT_FALSE(WaitForFrame(1000));
2185 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112186 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132187 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112188 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132189
2190 // Resolution higher than 360p. Encoder limits should be ignored.
2191 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2192 EXPECT_FALSE(WaitForFrame(1000));
2193 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112194 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132195 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112196 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132197 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112198 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132199 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112200 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132201
2202 // Resolution lower than 270p. The encoder limits for 270p should be used.
2203 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2204 EXPECT_FALSE(WaitForFrame(1000));
2205 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112206 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132207 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112208 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132209
2210 video_stream_encoder_->Stop();
2211}
2212
2213TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:512214 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2215 // Two streams, highest stream active.
2216 VideoEncoderConfig config;
2217 const int kNumStreams = 2;
2218 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2219 config.max_bitrate_bps = 0;
2220 config.simulcast_layers[0].active = false;
2221 config.simulcast_layers[1].active = true;
2222 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432223 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 09:39:512224 "VP8", /*max qp*/ 56, /*screencast*/ false,
2225 /*screenshare enabled*/ false);
2226 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2227
2228 // Default bitrate limits for 270p should be used.
2229 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2230 kDefaultLimits270p =
2231 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192232 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:512233 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2234 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112235 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 09:39:512236 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112237 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512238 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112239 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512240
2241 // Default bitrate limits for 360p should be used.
2242 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2243 kDefaultLimits360p =
2244 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192245 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:512246 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2247 EXPECT_FALSE(WaitForFrame(1000));
2248 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112249 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512250 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112251 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512252
2253 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2254 video_source_.IncomingCapturedFrame(
2255 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2256 EXPECT_FALSE(WaitForFrame(1000));
2257 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112258 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512259 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112260 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512261
2262 // Default bitrate limits for 540p should be used.
2263 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2264 kDefaultLimits540p =
2265 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192266 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 09:39:512267 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2268 EXPECT_FALSE(WaitForFrame(1000));
2269 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112270 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512271 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112272 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:512273
2274 video_stream_encoder_->Stop();
2275}
2276
2277TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:132278 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2279 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2280 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2281 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2282 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2283 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2284 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2285 fake_encoder_.SetResolutionBitrateLimits(
2286 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2287
2288 // Three streams, middle stream active.
2289 VideoEncoderConfig config;
2290 const int kNumStreams = 3;
2291 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2292 config.simulcast_layers[0].active = false;
2293 config.simulcast_layers[1].active = true;
2294 config.simulcast_layers[2].active = false;
2295 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432296 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132297 "VP8", /*max qp*/ 56, /*screencast*/ false,
2298 /*screenshare enabled*/ false);
2299 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2300
2301 // The encoder bitrate limits for 360p should be used.
2302 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2303 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112304 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132305 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112306 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132307 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112308 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132309
2310 // The encoder bitrate limits for 270p should be used.
2311 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2312 EXPECT_FALSE(WaitForFrame(1000));
2313 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112314 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132315 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112316 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132317
2318 video_stream_encoder_->Stop();
2319}
2320
2321TEST_F(VideoStreamEncoderTest,
2322 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2323 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2324 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2325 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2326 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2327 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2328 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2329 fake_encoder_.SetResolutionBitrateLimits(
2330 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2331
2332 // Three streams, lowest stream active.
2333 VideoEncoderConfig config;
2334 const int kNumStreams = 3;
2335 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2336 config.simulcast_layers[0].active = true;
2337 config.simulcast_layers[1].active = false;
2338 config.simulcast_layers[2].active = false;
2339 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432340 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132341 "VP8", /*max qp*/ 56, /*screencast*/ false,
2342 /*screenshare enabled*/ false);
2343 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2344
2345 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2346 // on lowest stream, limits for 270p should not be used
2347 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2348 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112349 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132350 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112351 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132352 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112353 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132354
2355 video_stream_encoder_->Stop();
2356}
2357
2358TEST_F(VideoStreamEncoderTest,
2359 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2360 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2361 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2362 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2363 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2364 fake_encoder_.SetResolutionBitrateLimits(
2365 {kEncoderLimits270p, kEncoderLimits360p});
2366 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2367
2368 // Two streams, highest stream active.
2369 VideoEncoderConfig config;
2370 const int kNumStreams = 2;
2371 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2372 config.simulcast_layers[0].active = false;
2373 config.simulcast_layers[1].active = true;
2374 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2375 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432376 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132377 "VP8", /*max qp*/ 56, /*screencast*/ false,
2378 /*screenshare enabled*/ false);
2379 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2380
2381 // The encoder bitrate limits for 270p should be used.
2382 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2383 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:112384 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 14:36:132385 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112386 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132387 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112388 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132389
2390 // The max configured bitrate is less than the encoder limit for 360p.
2391 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2392 EXPECT_FALSE(WaitForFrame(1000));
2393 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:112394 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132395 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 08:07:112396 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 14:36:132397
2398 video_stream_encoder_->Stop();
2399}
2400
mflodmancc3d4422017-08-03 15:27:512401TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 18:45:462402 EXPECT_TRUE(video_source_.has_sinks());
2403 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512404 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412405 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:462406 EXPECT_FALSE(video_source_.has_sinks());
2407 EXPECT_TRUE(new_video_source.has_sinks());
2408
mflodmancc3d4422017-08-03 15:27:512409 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462410}
2411
mflodmancc3d4422017-08-03 15:27:512412TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 18:45:462413 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512414 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 18:45:462415 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512416 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462417}
2418
Åsa Perssonc5a74ff2020-09-20 15:50:002419class ResolutionAlignmentTest
2420 : public VideoStreamEncoderTest,
2421 public ::testing::WithParamInterface<
2422 ::testing::tuple<int, std::vector<double>>> {
2423 public:
2424 ResolutionAlignmentTest()
2425 : requested_alignment_(::testing::get<0>(GetParam())),
2426 scale_factors_(::testing::get<1>(GetParam())) {}
2427
2428 protected:
2429 const int requested_alignment_;
2430 const std::vector<double> scale_factors_;
2431};
2432
2433INSTANTIATE_TEST_SUITE_P(
2434 AlignmentAndScaleFactors,
2435 ResolutionAlignmentTest,
2436 ::testing::Combine(
2437 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2438 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2439 std::vector<double>{-1.0, -1.0},
2440 std::vector<double>{-1.0, -1.0, -1.0},
2441 std::vector<double>{4.0, 2.0, 1.0},
2442 std::vector<double>{9999.0, -1.0, 1.0},
2443 std::vector<double>{3.99, 2.01, 1.0},
2444 std::vector<double>{4.9, 1.7, 1.25},
2445 std::vector<double>{10.0, 4.0, 3.0},
2446 std::vector<double>{1.75, 3.5},
2447 std::vector<double>{1.5, 2.5},
2448 std::vector<double>{1.3, 1.0})));
2449
2450TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2451 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 08:47:112452 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 15:50:002453 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2454 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2455
2456 // Fill config with the scaling factor by which to reduce encoding size.
2457 const int num_streams = scale_factors_.size();
2458 VideoEncoderConfig config;
2459 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2460 for (int i = 0; i < num_streams; ++i) {
2461 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2462 }
2463 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432464 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 15:50:002465 "VP8", /*max qp*/ 56, /*screencast*/ false,
2466 /*screenshare enabled*/ false);
2467 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2468
Henrik Boström381d1092020-05-12 16:49:072469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112470 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2471 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 15:50:002472 // Wait for all layers before triggering event.
2473 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 08:47:112474
2475 // On the 1st frame, we should have initialized the encoder and
2476 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 15:50:002477 int64_t timestamp_ms = kFrameIntervalMs;
2478 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2479 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:112480 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 08:47:112481
2482 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2483 // (It's up the to the encoder to potentially drop the previous frame,
2484 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 15:50:002485 timestamp_ms += kFrameIntervalMs;
2486 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2487 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:112488 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 15:50:002489
Asa Persson606d3cb2021-10-04 08:07:112490 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 15:50:002491 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2492 // Frame size should be a multiple of the requested alignment.
2493 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2494 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2495 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2496 // Aspect ratio should match.
2497 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2498 codec.height * codec.simulcastStream[i].width);
2499 }
Rasmus Brandt5cad55b2019-12-19 08:47:112500
2501 video_stream_encoder_->Stop();
2502}
2503
Jonathan Yubc771b72017-12-09 01:04:292504TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2505 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-14 06:25:222506 const int kWidth = 1280;
2507 const int kHeight = 720;
Jonathan Yubc771b72017-12-09 01:04:292508
2509 // We rely on the automatic resolution adaptation, but we handle framerate
2510 // adaptation manually by mocking the stats proxy.
2511 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-14 06:25:222512
Taylor Brandstetter49fcc102018-05-16 21:20:412513 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 16:49:072514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112515 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 21:20:412516 video_stream_encoder_->SetSource(&video_source_,
2517 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:392518 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:222519 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292520 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:222521 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2522
Jonathan Yubc771b72017-12-09 01:04:292523 // Adapt down as far as possible.
2524 rtc::VideoSinkWants last_wants;
2525 int64_t t = 1;
2526 int loop_count = 0;
2527 do {
2528 ++loop_count;
2529 last_wants = video_source_.sink_wants();
2530
2531 // Simulate the framerate we've been asked to adapt to.
2532 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2533 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2534 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2535 mock_stats.input_frame_rate = fps;
2536 stats_proxy_->SetMockStats(mock_stats);
2537
2538 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2539 sink_.WaitForEncodedFrame(t);
2540 t += frame_interval_ms;
2541
mflodmancc3d4422017-08-03 15:27:512542 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392543 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292544 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392545 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2546 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292547 } while (video_source_.sink_wants().max_pixel_count <
2548 last_wants.max_pixel_count ||
2549 video_source_.sink_wants().max_framerate_fps <
2550 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222551
Jonathan Yubc771b72017-12-09 01:04:292552 // Verify that we've adapted all the way down.
2553 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222554 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292555 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2556 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-14 06:25:222557 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:292558 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2559 *video_source_.last_sent_height());
2560 EXPECT_EQ(kMinBalancedFramerateFps,
2561 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222562
Jonathan Yubc771b72017-12-09 01:04:292563 // Adapt back up the same number of times we adapted down.
2564 for (int i = 0; i < loop_count - 1; ++i) {
2565 last_wants = video_source_.sink_wants();
2566
2567 // Simulate the framerate we've been asked to adapt to.
2568 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2569 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2570 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2571 mock_stats.input_frame_rate = fps;
2572 stats_proxy_->SetMockStats(mock_stats);
2573
2574 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2575 sink_.WaitForEncodedFrame(t);
2576 t += frame_interval_ms;
2577
Henrik Boström91aa7322020-04-28 10:24:332578 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392579 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292580 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392581 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2582 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292583 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2584 last_wants.max_pixel_count ||
2585 video_source_.sink_wants().max_framerate_fps >
2586 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222587 }
2588
Evan Shrubsole5cd7eb82020-05-25 12:08:392589 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-09 01:04:292590 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222591 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292592 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2593 EXPECT_EQ((loop_count - 1) * 2,
2594 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:222595
mflodmancc3d4422017-08-03 15:27:512596 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222597}
Rasmus Brandt5cad55b2019-12-19 08:47:112598
Evan Shrubsole2e2f6742020-05-14 08:41:152599TEST_F(VideoStreamEncoderTest,
2600 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 08:07:112601 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2602 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392603 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152604
2605 const int kFrameWidth = 1280;
2606 const int kFrameHeight = 720;
2607
2608 int64_t ntp_time = kFrameIntervalMs;
2609
2610 // Force an input frame rate to be available, or the adaptation call won't
2611 // know what framerate to adapt form.
2612 const int kInputFps = 30;
2613 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2614 stats.input_frame_rate = kInputFps;
2615 stats_proxy_->SetMockStats(stats);
2616
2617 video_source_.set_adaptation_enabled(true);
2618 video_stream_encoder_->SetSource(
2619 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 12:08:392620 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152621 video_source_.IncomingCapturedFrame(
2622 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2623 sink_.WaitForEncodedFrame(ntp_time);
2624 ntp_time += kFrameIntervalMs;
2625
2626 // Trigger CPU overuse.
2627 video_stream_encoder_->TriggerCpuOveruse();
2628 video_source_.IncomingCapturedFrame(
2629 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2630 sink_.WaitForEncodedFrame(ntp_time);
2631 ntp_time += kFrameIntervalMs;
2632
2633 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2634 EXPECT_EQ(std::numeric_limits<int>::max(),
2635 video_source_.sink_wants().max_pixel_count);
2636 // Some framerate constraint should be set.
2637 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2638 EXPECT_LT(restricted_fps, kInputFps);
2639 video_source_.IncomingCapturedFrame(
2640 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2641 sink_.WaitForEncodedFrame(ntp_time);
2642 ntp_time += 100;
2643
Henrik Boström2671dac2020-05-19 14:29:092644 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152645 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2646 // Give the encoder queue time to process the change in degradation preference
2647 // by waiting for an encoded frame.
2648 video_source_.IncomingCapturedFrame(
2649 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2650 sink_.WaitForEncodedFrame(ntp_time);
2651 ntp_time += kFrameIntervalMs;
2652
2653 video_stream_encoder_->TriggerQualityLow();
2654 video_source_.IncomingCapturedFrame(
2655 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2656 sink_.WaitForEncodedFrame(ntp_time);
2657 ntp_time += kFrameIntervalMs;
2658
2659 // Some resolution constraint should be set.
2660 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2661 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2662 kFrameWidth * kFrameHeight);
2663 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2664
2665 int pixel_count = video_source_.sink_wants().max_pixel_count;
2666 // Triggering a CPU underuse should not change the sink wants since it has
2667 // not been overused for resolution since we changed degradation preference.
2668 video_stream_encoder_->TriggerCpuUnderuse();
2669 video_source_.IncomingCapturedFrame(
2670 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2671 sink_.WaitForEncodedFrame(ntp_time);
2672 ntp_time += kFrameIntervalMs;
2673 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2674 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2675
Evan Shrubsole64469032020-06-11 08:45:292676 // Change the degradation preference back. CPU underuse should not adapt since
2677 // QP is most limited.
Henrik Boström2671dac2020-05-19 14:29:092678 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152679 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2680 video_source_.IncomingCapturedFrame(
2681 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2682 sink_.WaitForEncodedFrame(ntp_time);
2683 ntp_time += 100;
2684 // Resolution adaptations is gone after changing degradation preference.
2685 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2686 EXPECT_EQ(std::numeric_limits<int>::max(),
2687 video_source_.sink_wants().max_pixel_count);
2688 // The fps adaptation from above is now back.
2689 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2690
2691 // Trigger CPU underuse.
2692 video_stream_encoder_->TriggerCpuUnderuse();
2693 video_source_.IncomingCapturedFrame(
2694 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2695 sink_.WaitForEncodedFrame(ntp_time);
2696 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 08:45:292697 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2698
2699 // Trigger QP underuse, fps should return to normal.
2700 video_stream_encoder_->TriggerQualityHigh();
2701 video_source_.IncomingCapturedFrame(
2702 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2703 sink_.WaitForEncodedFrame(ntp_time);
2704 ntp_time += kFrameIntervalMs;
2705 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 08:41:152706
2707 video_stream_encoder_->Stop();
2708}
2709
mflodmancc3d4422017-08-03 15:27:512710TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 16:49:072711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112712 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392713 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 18:45:462714
sprangc5d62e22017-04-03 06:53:042715 const int kFrameWidth = 1280;
2716 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:042717
Åsa Persson8c1bf952018-09-13 08:42:192718 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 18:45:462719
kthelgason5e13d412016-12-01 11:59:512720 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042721 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522722 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042723 frame_timestamp += kFrameIntervalMs;
2724
perkj803d97f2016-11-01 18:45:462725 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512726 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462727 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042728 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522729 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042730 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 14:23:482731
asapersson0944a802017-04-07 07:57:582732 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-03 06:53:042733 // wanted resolution.
2734 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2735 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2736 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 08:42:192737 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042738
2739 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 18:45:462740 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 16:49:072741 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412742 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122743 // Give the encoder queue time to process the change in degradation preference
2744 // by waiting for an encoded frame.
2745 new_video_source.IncomingCapturedFrame(
2746 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2747 sink_.WaitForEncodedFrame(frame_timestamp);
2748 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042749 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 12:08:392750 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462751
sprangc5d62e22017-04-03 06:53:042752 // Force an input frame rate to be available, or the adaptation call won't
2753 // know what framerate to adapt form.
asapersson02465b82017-04-10 08:12:522754 const int kInputFps = 30;
sprangc5d62e22017-04-03 06:53:042755 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 08:12:522756 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-03 06:53:042757 stats_proxy_->SetMockStats(stats);
2758
mflodmancc3d4422017-08-03 15:27:512759 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462760 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042761 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522762 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042763 frame_timestamp += kFrameIntervalMs;
2764
2765 // Some framerate constraint should be set.
sprang84a37592017-02-10 15:04:272766 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-03 06:53:042767 EXPECT_EQ(std::numeric_limits<int>::max(),
2768 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522769 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-03 06:53:042770
asapersson02465b82017-04-10 08:12:522771 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 16:49:072772 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2773 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 10:42:122774 // Give the encoder queue time to process the change in degradation preference
2775 // by waiting for an encoded frame.
2776 new_video_source.IncomingCapturedFrame(
2777 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2778 sink_.WaitForEncodedFrame(frame_timestamp);
2779 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 12:08:392780 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-03 06:53:042781
mflodmancc3d4422017-08-03 15:27:512782 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042783 new_video_source.IncomingCapturedFrame(
2784 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522785 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042786 frame_timestamp += kFrameIntervalMs;
2787
2788 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 12:08:392789 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462790
2791 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072792 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412793 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 10:42:122794 // Give the encoder queue time to process the change in degradation preference
2795 // by waiting for an encoded frame.
2796 new_video_source.IncomingCapturedFrame(
2797 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2798 sink_.WaitForEncodedFrame(frame_timestamp);
2799 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042800 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2801 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 15:04:272802 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 08:42:192803 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042804
2805 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072806 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412807 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122808 // Give the encoder queue time to process the change in degradation preference
2809 // by waiting for an encoded frame.
2810 new_video_source.IncomingCapturedFrame(
2811 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2812 sink_.WaitForEncodedFrame(frame_timestamp);
2813 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042814 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2815 EXPECT_EQ(std::numeric_limits<int>::max(),
2816 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522817 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 18:45:462818
mflodmancc3d4422017-08-03 15:27:512819 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462820}
2821
mflodmancc3d4422017-08-03 15:27:512822TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 18:45:462825
asaperssonfab67072017-04-04 12:51:492826 const int kWidth = 1280;
2827 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492828 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522829 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 12:51:492830 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2831 EXPECT_FALSE(stats.bw_limited_resolution);
2832 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2833
2834 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512835 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492836 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522837 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 12:51:492838
2839 stats = stats_proxy_->GetStats();
2840 EXPECT_TRUE(stats.bw_limited_resolution);
2841 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2842
2843 // Trigger adapt up.
mflodmancc3d4422017-08-03 15:27:512844 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:492845 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522846 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 12:51:492847
2848 stats = stats_proxy_->GetStats();
2849 EXPECT_FALSE(stats.bw_limited_resolution);
2850 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2851 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2852
mflodmancc3d4422017-08-03 15:27:512853 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 12:51:492854}
2855
mflodmancc3d4422017-08-03 15:27:512856TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112858 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 12:51:492859
2860 const int kWidth = 1280;
2861 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492862 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522863 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 18:45:462864 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2865 EXPECT_FALSE(stats.cpu_limited_resolution);
2866 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2867
2868 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512869 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492870 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522871 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 18:45:462872
2873 stats = stats_proxy_->GetStats();
2874 EXPECT_TRUE(stats.cpu_limited_resolution);
2875 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2876
2877 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332878 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492879 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522880 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 18:45:462881
2882 stats = stats_proxy_->GetStats();
2883 EXPECT_FALSE(stats.cpu_limited_resolution);
2884 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 12:51:492885 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:462886
mflodmancc3d4422017-08-03 15:27:512887 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462888}
2889
mflodmancc3d4422017-08-03 15:27:512890TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112892 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112893
asaperssonfab67072017-04-04 12:51:492894 const int kWidth = 1280;
2895 const int kHeight = 720;
2896 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522897 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112898 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182899 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112900 EXPECT_FALSE(stats.cpu_limited_resolution);
2901 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2902
asaperssonfab67072017-04-04 12:51:492903 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512904 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492905 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522906 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112907 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182908 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112909 EXPECT_TRUE(stats.cpu_limited_resolution);
2910 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2911
2912 // Set new source with adaptation still enabled.
2913 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512914 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412915 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112916
asaperssonfab67072017-04-04 12:51:492917 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522918 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112919 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182920 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112921 EXPECT_TRUE(stats.cpu_limited_resolution);
2922 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2923
2924 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412925 video_stream_encoder_->SetSource(&new_video_source,
2926 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 09:44:112927
asaperssonfab67072017-04-04 12:51:492928 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522929 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112930 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182931 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112932 EXPECT_FALSE(stats.cpu_limited_resolution);
2933 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2934
2935 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 15:27:512936 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412937 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112938
asaperssonfab67072017-04-04 12:51:492939 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522940 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112941 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182942 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112943 EXPECT_TRUE(stats.cpu_limited_resolution);
2944 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2945
asaperssonfab67072017-04-04 12:51:492946 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332947 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492948 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522949 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 09:44:112950 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182951 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112952 EXPECT_FALSE(stats.cpu_limited_resolution);
2953 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:522954 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112955
mflodmancc3d4422017-08-03 15:27:512956 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112957}
2958
mflodmancc3d4422017-08-03 15:27:512959TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:112961 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112962
asaperssonfab67072017-04-04 12:51:492963 const int kWidth = 1280;
2964 const int kHeight = 720;
2965 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522966 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112967 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112968 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022969 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492970 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112971
2972 // Set new source with adaptation still enabled.
2973 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 21:20:412974 video_stream_encoder_->SetSource(&new_video_source,
2975 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112976
asaperssonfab67072017-04-04 12:51:492977 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522978 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112979 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112980 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022981 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492982 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112983
asaperssonfab67072017-04-04 12:51:492984 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512985 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492986 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522987 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112988 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112989 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022990 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492991 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112992
asaperssonfab67072017-04-04 12:51:492993 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412994 video_stream_encoder_->SetSource(&new_video_source,
2995 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112996
asaperssonfab67072017-04-04 12:51:492997 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522998 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112999 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:113000 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:023001 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:493002 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:113003
asapersson02465b82017-04-10 08:12:523004 // Disable resolution scaling.
mflodmancc3d4422017-08-03 15:27:513005 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413006 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:113007
asaperssonfab67072017-04-04 12:51:493008 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523009 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:113010 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:113011 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:023012 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:493013 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3014 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 09:44:113015
mflodmancc3d4422017-08-03 15:27:513016 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:113017}
3018
mflodmancc3d4422017-08-03 15:27:513019TEST_F(VideoStreamEncoderTest,
3020 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 16:49:073021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113022 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 12:29:123023
3024 const int kWidth = 1280;
3025 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 08:42:193026 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 12:29:123027 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 08:42:193028 video_source_.IncomingCapturedFrame(
3029 CreateFrame(timestamp_ms, kWidth, kHeight));
3030 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123031 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3032 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3033 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3034
3035 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:513036 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193037 timestamp_ms += kFrameIntervalMs;
3038 video_source_.IncomingCapturedFrame(
3039 CreateFrame(timestamp_ms, kWidth, kHeight));
3040 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123041 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3042 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3043 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3044
3045 // Trigger overuse.
mflodmancc3d4422017-08-03 15:27:513046 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193047 timestamp_ms += kFrameIntervalMs;
3048 video_source_.IncomingCapturedFrame(
3049 CreateFrame(timestamp_ms, kWidth, kHeight));
3050 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123051 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3052 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3053 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3054
Niels Möller4db138e2018-04-19 07:04:133055 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 12:29:123056 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:133057
3058 VideoEncoderConfig video_encoder_config;
3059 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3060 // Make format different, to force recreation of encoder.
3061 video_encoder_config.video_format.parameters["foo"] = "foo";
3062 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473063 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 08:42:193064 timestamp_ms += kFrameIntervalMs;
3065 video_source_.IncomingCapturedFrame(
3066 CreateFrame(timestamp_ms, kWidth, kHeight));
3067 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123068 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3069 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3070 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3071
mflodmancc3d4422017-08-03 15:27:513072 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 12:29:123073}
3074
mflodmancc3d4422017-08-03 15:27:513075TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 17:39:323076 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 16:49:073077 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113078 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 17:39:323079
3080 const int kWidth = 1280;
3081 const int kHeight = 720;
3082 int sequence = 1;
3083
3084 // Enable BALANCED preference, no initial limitation.
3085 test::FrameForwarder source;
3086 video_stream_encoder_->SetSource(&source,
3087 webrtc::DegradationPreference::BALANCED);
3088 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3089 WaitForEncodedFrame(sequence++);
3090 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3091 EXPECT_FALSE(stats.cpu_limited_resolution);
3092 EXPECT_FALSE(stats.cpu_limited_framerate);
3093 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3094
3095 // Trigger CPU overuse, should now adapt down.
3096 video_stream_encoder_->TriggerCpuOveruse();
3097 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3098 WaitForEncodedFrame(sequence++);
3099 stats = stats_proxy_->GetStats();
3100 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3101
3102 // Set new degradation preference should clear restrictions since we changed
3103 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 14:54:213104 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 17:39:323105 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3106 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3107 WaitForEncodedFrame(sequence++);
3108 stats = stats_proxy_->GetStats();
3109 EXPECT_FALSE(stats.cpu_limited_resolution);
3110 EXPECT_FALSE(stats.cpu_limited_framerate);
3111 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3112
3113 // Force an input frame rate to be available, or the adaptation call won't
3114 // know what framerate to adapt from.
3115 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3116 mock_stats.input_frame_rate = 30;
3117 stats_proxy_->SetMockStats(mock_stats);
3118 video_stream_encoder_->TriggerCpuOveruse();
3119 stats_proxy_->ResetMockStats();
3120 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3121 WaitForEncodedFrame(sequence++);
3122
3123 // We have now adapted once.
3124 stats = stats_proxy_->GetStats();
3125 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3126
3127 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 14:54:213128 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3129 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 17:39:323130 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3131 WaitForEncodedFrame(sequence++);
3132 stats = stats_proxy_->GetStats();
3133 EXPECT_FALSE(stats.cpu_limited_resolution);
3134 EXPECT_FALSE(stats.cpu_limited_framerate);
3135 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3136
3137 video_stream_encoder_->Stop();
3138}
3139
3140TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513141 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 16:49:073142 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113143 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 18:45:463144
asapersson0944a802017-04-07 07:57:583145 const int kWidth = 1280;
3146 const int kHeight = 720;
sprang84a37592017-02-10 15:04:273147 int sequence = 1;
perkj803d97f2016-11-01 18:45:463148
asaperssonfab67072017-04-04 12:51:493149 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523150 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463151 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 15:04:273152 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023153 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 15:04:273154 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3155
asapersson02465b82017-04-10 08:12:523156 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 15:27:513157 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:493158 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523159 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 15:04:273160 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 18:45:463161 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023162 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463163 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3164
3165 // Set new source with adaptation still enabled.
3166 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513167 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413168 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:463169
3170 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493171 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523172 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463173 stats = stats_proxy_->GetStats();
3174 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183175 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463176 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3177
sprangc5d62e22017-04-03 06:53:043178 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 15:27:513179 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413180 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 18:45:463181 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493182 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523183 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463184 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-03 06:53:043185 // Not adapted at first.
perkj803d97f2016-11-01 18:45:463186 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183187 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463188 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3189
sprangc5d62e22017-04-03 06:53:043190 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-16 06:40:183191 // know what framerate to adapt from.
sprangc5d62e22017-04-03 06:53:043192 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3193 mock_stats.input_frame_rate = 30;
3194 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513195 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043196 stats_proxy_->ResetMockStats();
3197
3198 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493199 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523200 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043201
3202 // Framerate now adapted.
3203 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:183204 EXPECT_FALSE(stats.cpu_limited_resolution);
3205 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043206 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3207
3208 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 21:20:413209 video_stream_encoder_->SetSource(&new_video_source,
3210 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-03 06:53:043211 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493212 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523213 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043214
3215 stats = stats_proxy_->GetStats();
3216 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023217 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043218 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3219
3220 // Try to trigger overuse. Should not succeed.
3221 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513222 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043223 stats_proxy_->ResetMockStats();
3224
3225 stats = stats_proxy_->GetStats();
3226 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023227 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043228 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3229
3230 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 15:27:513231 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413232 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 12:51:493233 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523234 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463235 stats = stats_proxy_->GetStats();
3236 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023237 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043238 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 18:45:463239
3240 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333241 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:493242 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523243 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463244 stats = stats_proxy_->GetStats();
3245 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023246 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043247 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3248
3249 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 15:27:513250 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413251 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:043252 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493253 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523254 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043255 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 07:01:023256 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-03 06:53:043257 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023258 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043259 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3260
3261 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333262 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:043263 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493264 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523265 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043266 stats = stats_proxy_->GetStats();
3267 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023268 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043269 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:523270 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:463271
mflodmancc3d4422017-08-03 15:27:513272 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:463273}
3274
mflodmancc3d4422017-08-03 15:27:513275TEST_F(VideoStreamEncoderTest,
3276 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 12:51:493277 const int kWidth = 1280;
3278 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073279 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113280 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 09:44:113281
asaperssonfab67072017-04-04 12:51:493282 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 12:08:393283 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 09:44:113284
asaperssonfab67072017-04-04 12:51:493285 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523286 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:113287
asaperssonfab67072017-04-04 12:51:493288 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513289 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 11:59:513290
asaperssonfab67072017-04-04 12:51:493291 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523292 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 11:59:513293
kthelgason876222f2016-11-29 09:44:113294 // Expect a scale down.
3295 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 12:51:493296 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 09:44:113297
asapersson02465b82017-04-10 08:12:523298 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 09:44:113299 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513300 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413301 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:113302
asaperssonfab67072017-04-04 12:51:493303 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513304 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:493305 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523306 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:113307
asaperssonfab67072017-04-04 12:51:493308 // Expect no scaling.
sprangc5d62e22017-04-03 06:53:043309 EXPECT_EQ(std::numeric_limits<int>::max(),
3310 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113311
asaperssonfab67072017-04-04 12:51:493312 // Trigger scale up.
mflodmancc3d4422017-08-03 15:27:513313 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:493314 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523315 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:113316
asapersson02465b82017-04-10 08:12:523317 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-03 06:53:043318 EXPECT_EQ(std::numeric_limits<int>::max(),
3319 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113320
mflodmancc3d4422017-08-03 15:27:513321 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:113322}
3323
mflodmancc3d4422017-08-03 15:27:513324TEST_F(VideoStreamEncoderTest,
3325 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523326 const int kWidth = 1280;
3327 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073328 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113329 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523330
Taylor Brandstetter49fcc102018-05-16 21:20:413331 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523332 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513333 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413334 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523335
3336 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523337 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393338 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523339 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3340 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3341
3342 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513343 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393344 EXPECT_THAT(source.sink_wants(),
3345 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523346 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3347 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3348 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3349
3350 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 15:27:513351 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 08:12:523352 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3353 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3354 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3355
mflodmancc3d4422017-08-03 15:27:513356 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523357}
3358
mflodmancc3d4422017-08-03 15:27:513359TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223360 const int kWidth = 1280;
3361 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073362 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113363 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223364
Taylor Brandstetter49fcc102018-05-16 21:20:413365 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223366 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413367 video_stream_encoder_->SetSource(&source,
3368 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223369 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3370 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393371 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223372
3373 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513374 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:393375 EXPECT_THAT(source.sink_wants(),
3376 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223377 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3378 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3379 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3380
3381 // Trigger adapt down for same input resolution, expect no change.
3382 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3383 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:513384 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223385 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3386 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3387 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3388
3389 // Trigger adapt down for larger input resolution, expect no change.
3390 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3391 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:513392 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223393 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3394 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3395 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3396
mflodmancc3d4422017-08-03 15:27:513397 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223398}
3399
mflodmancc3d4422017-08-03 15:27:513400TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 20:55:383401 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3402 const int kWidth = 640;
3403 const int kHeight = 360;
3404 const int64_t kFrameIntervalMs = 150;
3405 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113406 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 20:55:383407
3408 // Enable BALANCED preference, no initial limitation.
3409 AdaptingFrameForwarder source(&time_controller_);
3410 source.set_adaptation_enabled(true);
3411 video_stream_encoder_->SetSource(&source,
3412 webrtc::DegradationPreference::BALANCED);
3413
3414 int64_t timestamp_ms = kFrameIntervalMs;
3415 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3416 sink_.WaitForEncodedFrame(kWidth, kHeight);
3417 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3418 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3419 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3420 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3421
3422 // Trigger adapt down, expect reduced fps (640x360@15fps).
3423 video_stream_encoder_->TriggerQualityLow();
3424 timestamp_ms += kFrameIntervalMs;
3425 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3426 sink_.WaitForEncodedFrame(timestamp_ms);
3427 EXPECT_THAT(source.sink_wants(),
3428 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3429 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3430 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3431 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3432
3433 // Source requests 270p, expect reduced resolution (480x270@15fps).
3434 source.OnOutputFormatRequest(480, 270);
3435 timestamp_ms += kFrameIntervalMs;
3436 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3437 WaitForEncodedFrame(480, 270);
3438 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3439
3440 // Trigger adapt down, expect reduced fps (480x270@10fps).
3441 video_stream_encoder_->TriggerQualityLow();
3442 timestamp_ms += kFrameIntervalMs;
3443 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3444 sink_.WaitForEncodedFrame(timestamp_ms);
3445 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3447 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3448 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3449
3450 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3451 source.OnOutputFormatRequest(320, 180);
3452 timestamp_ms += kFrameIntervalMs;
3453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3454 WaitForEncodedFrame(320, 180);
3455 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3456
3457 // Trigger adapt down, expect reduced fps (320x180@7fps).
3458 video_stream_encoder_->TriggerQualityLow();
3459 timestamp_ms += kFrameIntervalMs;
3460 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3461 sink_.WaitForEncodedFrame(timestamp_ms);
3462 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3463 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3464 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3465 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3466
3467 // Source requests VGA, expect increased resolution (640x360@7fps).
3468 source.OnOutputFormatRequest(640, 360);
3469 timestamp_ms += kFrameIntervalMs;
3470 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3471 WaitForEncodedFrame(timestamp_ms);
3472 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3473
3474 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3475 video_stream_encoder_->TriggerQualityHigh();
3476 timestamp_ms += kFrameIntervalMs;
3477 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3478 WaitForEncodedFrame(timestamp_ms);
3479 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3482 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3483
3484 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3485 video_stream_encoder_->TriggerQualityHigh();
3486 timestamp_ms += kFrameIntervalMs;
3487 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3488 WaitForEncodedFrame(timestamp_ms);
3489 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3490 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3491 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3492 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3493
3494 // Trigger adapt up, expect increased fps (640x360@maxfps).
3495 video_stream_encoder_->TriggerQualityHigh();
3496 timestamp_ms += kFrameIntervalMs;
3497 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3498 WaitForEncodedFrame(timestamp_ms);
3499 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3500 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3502 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3503
3504 video_stream_encoder_->Stop();
3505}
3506
3507TEST_F(VideoStreamEncoderTest,
3508 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3509 const int kWidth = 1280;
3510 const int kHeight = 720;
3511 const int64_t kFrameIntervalMs = 150;
3512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113513 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 20:55:383514
3515 // Enable BALANCED preference, no initial limitation.
3516 AdaptingFrameForwarder source(&time_controller_);
3517 source.set_adaptation_enabled(true);
3518 video_stream_encoder_->SetSource(&source,
3519 webrtc::DegradationPreference::BALANCED);
3520
3521 int64_t timestamp_ms = kFrameIntervalMs;
3522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3523 sink_.WaitForEncodedFrame(kWidth, kHeight);
3524 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3525 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3527 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3528
3529 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3530 video_stream_encoder_->TriggerQualityLow();
3531 timestamp_ms += kFrameIntervalMs;
3532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3533 sink_.WaitForEncodedFrame(timestamp_ms);
3534 EXPECT_THAT(source.sink_wants(),
3535 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3538 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3539
3540 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3541 video_stream_encoder_->TriggerQualityLow();
3542 timestamp_ms += kFrameIntervalMs;
3543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3544 sink_.WaitForEncodedFrame(timestamp_ms);
3545 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3548 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3549
3550 // Trigger adapt down, expect reduced fps (640x360@15fps).
3551 video_stream_encoder_->TriggerQualityLow();
3552 timestamp_ms += kFrameIntervalMs;
3553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3554 WaitForEncodedFrame(timestamp_ms);
3555 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3558 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3559
3560 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3561 source.OnOutputFormatRequest(320, 180);
3562 timestamp_ms += kFrameIntervalMs;
3563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3564 WaitForEncodedFrame(320, 180);
3565 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3566 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3567
3568 // Trigger adapt down, expect reduced fps (320x180@7fps).
3569 video_stream_encoder_->TriggerCpuOveruse();
3570 timestamp_ms += kFrameIntervalMs;
3571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3572 WaitForEncodedFrame(timestamp_ms);
3573 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3576 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3577 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3578 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3579 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3580
3581 // Source requests HD, expect increased resolution (640x360@7fps).
3582 source.OnOutputFormatRequest(1280, 720);
3583 timestamp_ms += kFrameIntervalMs;
3584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3585 WaitForEncodedFrame(timestamp_ms);
3586 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3588
3589 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3590 video_stream_encoder_->TriggerCpuUnderuse();
3591 timestamp_ms += kFrameIntervalMs;
3592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3593 WaitForEncodedFrame(timestamp_ms);
3594 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3597 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3598 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3599 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3600 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3601
3602 // Trigger adapt up, expect increased fps (640x360@maxfps).
3603 video_stream_encoder_->TriggerQualityHigh();
3604 video_stream_encoder_->TriggerCpuUnderuse();
3605 timestamp_ms += kFrameIntervalMs;
3606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3607 WaitForEncodedFrame(timestamp_ms);
3608 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3609 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3610 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3611 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3612 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3613 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3614 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3615
3616 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3617 video_stream_encoder_->TriggerQualityHigh();
3618 video_stream_encoder_->TriggerCpuUnderuse();
3619 timestamp_ms += kFrameIntervalMs;
3620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3621 WaitForEncodedFrame(timestamp_ms);
3622 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3624 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3625 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3626 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3627 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3628 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3629
3630 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3631 video_stream_encoder_->TriggerQualityHigh();
3632 video_stream_encoder_->TriggerCpuUnderuse();
3633 timestamp_ms += kFrameIntervalMs;
3634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3635 WaitForEncodedFrame(timestamp_ms);
3636 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3637 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3638 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3639 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3640 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3641 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3642 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3643
3644 video_stream_encoder_->Stop();
3645}
3646
3647TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513648 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523649 const int kWidth = 1280;
3650 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073651 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113652 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523653
Taylor Brandstetter49fcc102018-05-16 21:20:413654 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523655 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513656 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413657 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523658
3659 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523660 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393661 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3663 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3664
3665 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333666 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393667 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523668 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3669 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3670
mflodmancc3d4422017-08-03 15:27:513671 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523672}
3673
mflodmancc3d4422017-08-03 15:27:513674TEST_F(VideoStreamEncoderTest,
3675 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 08:12:523676 const int kWidth = 1280;
3677 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113679 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523680
Taylor Brandstetter49fcc102018-05-16 21:20:413681 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523682 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513683 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413684 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 08:12:523685
3686 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523687 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393688 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183689 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523690 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3691
3692 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333693 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393694 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183695 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523696 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3697
mflodmancc3d4422017-08-03 15:27:513698 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523699}
3700
mflodmancc3d4422017-08-03 15:27:513701TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223702 const int kWidth = 1280;
3703 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073704 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113705 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223706
Taylor Brandstetter49fcc102018-05-16 21:20:413707 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223708 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413709 video_stream_encoder_->SetSource(&source,
3710 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223711
3712 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3713 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393714 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223715 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3716 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3717 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3718
3719 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513720 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393721 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3723 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3725
mflodmancc3d4422017-08-03 15:27:513726 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223727}
3728
mflodmancc3d4422017-08-03 15:27:513729TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-16 06:40:183730 const int kWidth = 1280;
3731 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073732 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113733 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:183734
Taylor Brandstetter49fcc102018-05-16 21:20:413735 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-16 06:40:183736 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413737 video_stream_encoder_->SetSource(&source,
3738 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-16 06:40:183739
3740 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3741 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393742 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183743 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3744 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3745 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3746
3747 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513748 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393749 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183750 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3751 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3752 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3753
mflodmancc3d4422017-08-03 15:27:513754 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183755}
3756
mflodmancc3d4422017-08-03 15:27:513757TEST_F(VideoStreamEncoderTest,
3758 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523759 const int kWidth = 1280;
3760 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073761 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113762 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 08:12:523763
Taylor Brandstetter49fcc102018-05-16 21:20:413764 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233765 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 08:12:523766 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513767 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413768 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523769
3770 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523771 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393772 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523773 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3774 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3775
3776 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513777 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 08:12:523778 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523779 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393780 EXPECT_THAT(source.sink_wants(),
3781 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523782 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3783 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3784
3785 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513786 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393787 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523788 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3789 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3790 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3791
mflodmancc3d4422017-08-03 15:27:513792 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523793}
3794
mflodmancc3d4422017-08-03 15:27:513795TEST_F(VideoStreamEncoderTest,
3796 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-16 06:40:183797 const int kWidth = 1280;
3798 const int kHeight = 720;
3799 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 16:49:073800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:183802
3803 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3804 stats.input_frame_rate = kInputFps;
3805 stats_proxy_->SetMockStats(stats);
3806
Taylor Brandstetter49fcc102018-05-16 21:20:413807 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-16 06:40:183808 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3809 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393810 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183811
3812 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513813 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-16 06:40:183814 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3815 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393816 EXPECT_THAT(video_source_.sink_wants(),
3817 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-16 06:40:183818
Taylor Brandstetter49fcc102018-05-16 21:20:413819 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-16 06:40:183820 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 14:29:093821 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:413822 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:123823 // Give the encoder queue time to process the change in degradation preference
3824 // by waiting for an encoded frame.
3825 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3826 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 12:08:393827 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183828
3829 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 15:27:513830 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 10:42:123831 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3832 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 12:08:393833 EXPECT_THAT(new_video_source.sink_wants(),
3834 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-16 06:40:183835
3836 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513837 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393838 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183839
mflodmancc3d4422017-08-03 15:27:513840 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183841}
3842
mflodmancc3d4422017-08-03 15:27:513843TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 08:47:313844 const int kWidth = 1280;
3845 const int kHeight = 720;
3846 const size_t kNumFrames = 10;
3847
Henrik Boström381d1092020-05-12 16:49:073848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113849 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 11:59:513850
asaperssond0de2952017-04-21 08:47:313851 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 15:58:543852 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 08:47:313853 video_source_.set_adaptation_enabled(true);
3854
3855 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3857
3858 int downscales = 0;
3859 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 08:42:193860 video_source_.IncomingCapturedFrame(
3861 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3862 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 08:47:313863
asaperssonfab67072017-04-04 12:51:493864 // Trigger scale down.
asaperssond0de2952017-04-21 08:47:313865 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 15:27:513866 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-03 06:53:043867 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 08:47:313868
3869 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3870 ++downscales;
3871
3872 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3873 EXPECT_EQ(downscales,
3874 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3875 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 11:59:513876 }
mflodmancc3d4422017-08-03 15:27:513877 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313878}
3879
mflodmancc3d4422017-08-03 15:27:513880TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:313881 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3882 const int kWidth = 1280;
3883 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073884 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113885 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:313886
Taylor Brandstetter49fcc102018-05-16 21:20:413887 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233888 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:313889 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513890 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413891 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:313892
Åsa Persson8c1bf952018-09-13 08:42:193893 int64_t timestamp_ms = kFrameIntervalMs;
3894 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523895 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393896 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313897 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3898 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3899
3900 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513901 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193902 timestamp_ms += kFrameIntervalMs;
3903 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3904 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393905 EXPECT_THAT(source.sink_wants(),
3906 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313907 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3908 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3909
3910 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333911 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193912 timestamp_ms += kFrameIntervalMs;
3913 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523914 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393915 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313916 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3917 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3918
3919 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513920 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193921 timestamp_ms += kFrameIntervalMs;
3922 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3923 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393924 EXPECT_THAT(source.sink_wants(),
3925 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313926 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3927 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3928
3929 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333930 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193931 timestamp_ms += kFrameIntervalMs;
3932 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-16 06:40:183933 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393934 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313935 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3936 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3937
mflodmancc3d4422017-08-03 15:27:513938 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313939}
3940
mflodmancc3d4422017-08-03 15:27:513941TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-14 06:25:223942 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3943 const int kWidth = 1280;
3944 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:113946 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223947
Taylor Brandstetter49fcc102018-05-16 21:20:413948 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233949 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:223950 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:413951 video_stream_encoder_->SetSource(&source,
3952 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223953
Åsa Persson8c1bf952018-09-13 08:42:193954 int64_t timestamp_ms = kFrameIntervalMs;
3955 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223956 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393957 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223958 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3959 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3960
3961 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513962 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193963 timestamp_ms += kFrameIntervalMs;
3964 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3965 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393966 EXPECT_THAT(source.sink_wants(),
3967 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223968 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3969 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3970
3971 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513972 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193973 timestamp_ms += kFrameIntervalMs;
3974 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223975 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393976 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223977 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3978 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3979
3980 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513981 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193982 timestamp_ms += kFrameIntervalMs;
3983 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3984 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393985 EXPECT_THAT(source.sink_wants(),
3986 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223987 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3988 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3989
3990 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513991 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193992 timestamp_ms += kFrameIntervalMs;
3993 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223994 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393995 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223996 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3997 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3998
mflodmancc3d4422017-08-03 15:27:513999 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224000}
4001
Sergey Silkin41c650b2019-10-14 11:12:194002TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4003 fake_encoder_.SetResolutionBitrateLimits(
4004 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4005
Henrik Boström381d1092020-05-12 16:49:074006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074007 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4008 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4009 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4010 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194011
4012 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234013 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:194014 source.set_adaptation_enabled(true);
4015 video_stream_encoder_->SetSource(
4016 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4017
4018 // Insert 720p frame.
4019 int64_t timestamp_ms = kFrameIntervalMs;
4020 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4021 WaitForEncodedFrame(1280, 720);
4022
4023 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 16:49:074024 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074025 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4026 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4027 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4028 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194029 video_stream_encoder_->TriggerQualityLow();
4030
4031 // Insert 720p frame. It should be downscaled and encoded.
4032 timestamp_ms += kFrameIntervalMs;
4033 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4034 WaitForEncodedFrame(960, 540);
4035
4036 // Trigger adapt up. Higher resolution should not be requested duo to lack
4037 // of bitrate.
4038 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:394039 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 11:12:194040
4041 // Increase bitrate.
Henrik Boström381d1092020-05-12 16:49:074042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074043 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4044 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4045 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4046 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194047
4048 // Trigger adapt up. Higher resolution should be requested.
4049 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:394050 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 11:12:194051
4052 video_stream_encoder_->Stop();
4053}
4054
4055TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4056 fake_encoder_.SetResolutionBitrateLimits(
4057 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4058
4059 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 16:49:074060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074061 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4062 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4063 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4064 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194065
4066 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234067 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:194068 source.set_adaptation_enabled(true);
4069 video_stream_encoder_->SetSource(
4070 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4071
4072 // Insert 720p frame. It should be dropped and lower resolution should be
4073 // requested.
4074 int64_t timestamp_ms = kFrameIntervalMs;
4075 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4076 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:094077 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 11:12:194078
4079 // Insert 720p frame. It should be downscaled and encoded.
4080 timestamp_ms += kFrameIntervalMs;
4081 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4082 WaitForEncodedFrame(960, 540);
4083
4084 video_stream_encoder_->Stop();
4085}
4086
Åsa Perssonb67c44c2019-09-24 13:25:324087class BalancedDegradationTest : public VideoStreamEncoderTest {
4088 protected:
4089 void SetupTest() {
4090 // Reset encoder for field trials to take effect.
4091 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:114092 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 13:25:324093
4094 // Enable BALANCED preference.
4095 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 13:13:044096 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4097 }
4098
Asa Persson606d3cb2021-10-04 08:07:114099 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 16:49:074100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114101 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 13:25:324102 }
4103
Åsa Persson45b176f2019-09-30 09:19:054104 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 13:25:324105 timestamp_ms_ += kFrameIntervalMs;
4106 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 09:19:054107 }
4108
4109 void InsertFrameAndWaitForEncoded() {
4110 InsertFrame();
Åsa Perssonb67c44c2019-09-24 13:25:324111 sink_.WaitForEncodedFrame(timestamp_ms_);
4112 }
4113
4114 const int kWidth = 640; // pixels:640x360=230400
4115 const int kHeight = 360;
4116 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4117 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:234118 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 13:25:324119};
4120
Evan Shrubsolea1c77f62020-08-10 09:01:064121TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 13:25:324122 test::ScopedFieldTrials field_trials(
4123 "WebRTC-Video-BalancedDegradationSettings/"
4124 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4125 SetupTest();
4126
4127 // Force input frame rate.
4128 const int kInputFps = 24;
4129 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4130 stats.input_frame_rate = kInputFps;
4131 stats_proxy_->SetMockStats(stats);
4132
Åsa Persson45b176f2019-09-30 09:19:054133 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394134 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324135
Evan Shrubsolea1c77f62020-08-10 09:01:064136 // Trigger adapt down, expect scaled down framerate and resolution,
4137 // since Fps diff (input-requested:0) < threshold.
4138 video_stream_encoder_->TriggerQualityLow();
4139 EXPECT_THAT(source_.sink_wants(),
4140 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 13:25:324141
4142 video_stream_encoder_->Stop();
4143}
4144
Evan Shrubsolea1c77f62020-08-10 09:01:064145TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 13:25:324146 test::ScopedFieldTrials field_trials(
4147 "WebRTC-Video-BalancedDegradationSettings/"
4148 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4149 SetupTest();
4150
4151 // Force input frame rate.
4152 const int kInputFps = 25;
4153 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4154 stats.input_frame_rate = kInputFps;
4155 stats_proxy_->SetMockStats(stats);
4156
Åsa Persson45b176f2019-09-30 09:19:054157 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394158 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324159
Evan Shrubsolea1c77f62020-08-10 09:01:064160 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4161 // Fps diff (input-requested:1) == threshold.
4162 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:394163 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 13:25:324164
4165 video_stream_encoder_->Stop();
4166}
4167
4168TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4169 test::ScopedFieldTrials field_trials(
4170 "WebRTC-Video-BalancedDegradationSettings/"
4171 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4172 SetupTest();
4173
4174 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4175
Åsa Persson45b176f2019-09-30 09:19:054176 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394177 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324178
4179 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4180 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:394181 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 13:25:324182
4183 video_stream_encoder_->Stop();
4184}
4185
Åsa Perssonccfb3402019-09-25 13:13:044186TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324187 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 15:26:394188 "WebRTC-Video-BalancedDegradationSettings/"
4189 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 13:13:044190 SetupTest();
Åsa Persson1b247f12019-08-14 15:26:394191
Asa Persson606d3cb2021-10-04 08:07:114192 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4193 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4194 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 15:26:394195
Åsa Persson45b176f2019-09-30 09:19:054196 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394197 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 15:26:394198 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4199
4200 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4201 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054202 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394203 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 15:26:394204 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4205
4206 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4207 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054208 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544209 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:394210 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4211
Åsa Persson30ab0152019-08-27 10:22:334212 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4213 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054214 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544215 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 13:13:044216 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 10:22:334217 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4218
4219 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 15:26:394220 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054221 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334222 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 15:26:394223
Åsa Persson30ab0152019-08-27 10:22:334224 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114225 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 15:26:394226 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054227 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:044228 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 10:22:334229 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4230
4231 video_stream_encoder_->Stop();
4232}
4233
Åsa Perssonccfb3402019-09-25 13:13:044234TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 09:19:054235 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4236 test::ScopedFieldTrials field_trials(
4237 "WebRTC-Video-BalancedDegradationSettings/"
4238 "pixels:57600|129600|230400,fps:7|24|24/");
4239 SetupTest();
Asa Persson606d3cb2021-10-04 08:07:114240 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 09:19:054241
Evan Shrubsole5cd7eb82020-05-25 12:08:394242 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 09:19:054243
4244 // Insert frame, expect scaled down:
4245 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4246 InsertFrame();
4247 EXPECT_FALSE(WaitForFrame(1000));
4248 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4249 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4250
4251 // Insert frame, expect scaled down:
4252 // resolution (320x180@24fps).
4253 InsertFrame();
4254 EXPECT_FALSE(WaitForFrame(1000));
4255 EXPECT_LT(source_.sink_wants().max_pixel_count,
4256 source_.last_wants().max_pixel_count);
4257 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4258
4259 // Frame should not be dropped (min pixels per frame reached).
4260 InsertFrameAndWaitForEncoded();
4261
4262 video_stream_encoder_->Stop();
4263}
4264
4265TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334266 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324267 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:334268 "WebRTC-Video-BalancedDegradationSettings/"
4269 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044270 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334271
Asa Persson606d3cb2021-10-04 08:07:114272 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4273 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4274 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 10:22:334275
Åsa Persson45b176f2019-09-30 09:19:054276 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394277 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334278 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4279
4280 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4281 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054282 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394283 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334284 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4285
4286 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4287 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054288 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544289 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334290 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4291
4292 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4293 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054294 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544295 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:394296 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4297
Åsa Persson30ab0152019-08-27 10:22:334298 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4299 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054300 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544301 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334302 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4303
4304 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4305 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054306 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334307 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4308
4309 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114310 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334311 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054312 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544313 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334314 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4315
4316 video_stream_encoder_->Stop();
4317}
4318
Åsa Perssonccfb3402019-09-25 13:13:044319TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334320 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324321 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:334322 "WebRTC-Video-BalancedDegradationSettings/"
4323 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044324 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334325
Asa Persson606d3cb2021-10-04 08:07:114326 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4327 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4328 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4329 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4330 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334331
Åsa Persson45b176f2019-09-30 09:19:054332 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394333 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334334 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4335
4336 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4337 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054338 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394339 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334340 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4341
4342 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4343 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054344 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544345 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334346 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4347
4348 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4349 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054350 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544351 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334352 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4353
4354 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4355 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054356 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334357 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4358
4359 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114360 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334361 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054362 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544363 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334364 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4365
4366 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114367 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 10:22:334368 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054369 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334370 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4371
4372 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 08:07:114373 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 10:22:334374 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054375 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544376 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334377 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4378
Åsa Persson1b247f12019-08-14 15:26:394379 video_stream_encoder_->Stop();
4380}
4381
mflodmancc3d4422017-08-03 15:27:514382TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:314383 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4384 const int kWidth = 1280;
4385 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:074386 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114387 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:314388
Taylor Brandstetter49fcc102018-05-16 21:20:414389 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234390 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:314391 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:514392 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414393 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:314394
Åsa Persson8c1bf952018-09-13 08:42:194395 int64_t timestamp_ms = kFrameIntervalMs;
4396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524397 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:394398 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314399 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4400 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4401 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4402 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4403
4404 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 15:27:514405 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194406 timestamp_ms += kFrameIntervalMs;
4407 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4408 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:394409 EXPECT_THAT(source.sink_wants(),
4410 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:314411 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4412 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4413 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4414 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4415
4416 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 15:27:514417 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194418 timestamp_ms += kFrameIntervalMs;
4419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4420 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544421 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314422 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4424 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4425 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4426
Jonathan Yubc771b72017-12-09 01:04:294427 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 15:27:514428 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194429 timestamp_ms += kFrameIntervalMs;
4430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4431 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544432 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314433 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4434 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294435 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:314436 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4437
Jonathan Yubc771b72017-12-09 01:04:294438 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 15:27:514439 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194440 timestamp_ms += kFrameIntervalMs;
4441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4442 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544443 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294444 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 08:47:314445 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4446 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4447 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4448 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4449
Jonathan Yubc771b72017-12-09 01:04:294450 // Trigger quality adapt down, expect no change (min resolution reached).
4451 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194452 timestamp_ms += kFrameIntervalMs;
4453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4454 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544455 EXPECT_THAT(source.sink_wants(), FpsMax());
4456 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-09 01:04:294457 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4458 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4459 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4460 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4461
Evan Shrubsole64469032020-06-11 08:45:294462 // Trigger quality adapt up, expect upscaled resolution (480x270).
4463 video_stream_encoder_->TriggerQualityHigh();
4464 timestamp_ms += kFrameIntervalMs;
4465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4466 WaitForEncodedFrame(timestamp_ms);
4467 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4468 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4469 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4470 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4471 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4472
4473 // Trigger quality and cpu adapt up since both are most limited, expect
4474 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 10:24:334475 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294476 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194477 timestamp_ms += kFrameIntervalMs;
4478 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4479 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544480 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294481 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4482 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4483 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294484 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:294485
Evan Shrubsole64469032020-06-11 08:45:294486 // Trigger quality and cpu adapt up since both are most limited, expect
4487 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 10:24:334488 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294489 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194490 timestamp_ms += kFrameIntervalMs;
4491 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4492 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544493 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314494 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 08:45:294495 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 08:47:314496 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:294497 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4498 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314499
Evan Shrubsole64469032020-06-11 08:45:294500 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4501 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 10:24:334502 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:194503 timestamp_ms += kFrameIntervalMs;
4504 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4505 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544506 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 08:47:314507 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294509 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294510 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314511
4512 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 15:27:514513 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194514 timestamp_ms += kFrameIntervalMs;
4515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524516 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:544517 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:394518 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314519 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4520 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294521 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294522 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 11:59:514523
mflodmancc3d4422017-08-03 15:27:514524 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 11:59:514525}
4526
mflodmancc3d4422017-08-03 15:27:514527TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 12:51:494528 const int kWidth = 640;
4529 const int kHeight = 360;
perkj803d97f2016-11-01 18:45:464530
Henrik Boström381d1092020-05-12 16:49:074531 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114532 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:524533
perkj803d97f2016-11-01 18:45:464534 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494535 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524536 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 18:45:464537 }
4538
mflodmancc3d4422017-08-03 15:27:514539 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:464540 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494541 video_source_.IncomingCapturedFrame(CreateFrame(
4542 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524543 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 18:45:464544 }
4545
mflodmancc3d4422017-08-03 15:27:514546 video_stream_encoder_->Stop();
4547 video_stream_encoder_.reset();
perkj803d97f2016-11-01 18:45:464548 stats_proxy_.reset();
sprangf8ee65e2017-02-28 16:49:334549
Ying Wangef3998f2019-12-09 12:06:534550 EXPECT_METRIC_EQ(
4551 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4552 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 18:45:464553 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4554}
4555
mflodmancc3d4422017-08-03 15:27:514556TEST_F(VideoStreamEncoderTest,
4557 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 16:49:074558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114559 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 09:01:064560 const int kWidth = 640;
4561 const int kHeight = 360;
4562
Taylor Brandstetter49fcc102018-05-16 21:20:414563 video_stream_encoder_->SetSource(&video_source_,
4564 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 09:01:064565
4566 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4567 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524568 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 09:01:064569 }
4570
mflodmancc3d4422017-08-03 15:27:514571 video_stream_encoder_->Stop();
4572 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 09:01:064573 stats_proxy_.reset();
4574
4575 EXPECT_EQ(0,
4576 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4577}
4578
Per Kjellanderdcef6412020-10-07 13:09:054579TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4580 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034581 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:054582 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 14:24:024583
4584 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 10:32:224585 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 08:07:114586 SimulcastRateAllocator(fake_encoder_.config())
4587 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 13:16:284588 kDefaultFps));
sprang57c2fff2017-01-16 14:24:024589
Henrik Boström381d1092020-05-12 16:49:074590 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114591 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 14:24:024592
sprang57c2fff2017-01-16 14:24:024593 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234594 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4595 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054596 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4597 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4598
Erik Språngd7329ca2019-02-21 20:19:534599 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 15:44:424600 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 12:31:234601 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024602
Per Kjellanderdcef6412020-10-07 13:09:054603 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 14:24:024604 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234605 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4606 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054607 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 12:31:234608 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024609
Per Kjellanderdcef6412020-10-07 13:09:054610 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 12:31:234611 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 09:28:414612 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 20:19:534613 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234614 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4615 WaitForEncodedFrame(CurrentTimeMs());
4616 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 20:19:534617 }
Per Kjellanderdcef6412020-10-07 13:09:054618 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 20:19:534619
mflodmancc3d4422017-08-03 15:27:514620 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 14:24:024621}
4622
Per Kjellanderf86cf4c2020-12-30 14:27:354623TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 15:53:224624 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034625 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:224626 kVideoLayersAllocation);
4627
4628 const int kDefaultFps = 30;
4629
4630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114631 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:224632
4633 video_source_.IncomingCapturedFrame(
4634 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4635 WaitForEncodedFrame(CurrentTimeMs());
4636 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4637 VideoLayersAllocation last_layer_allocation =
4638 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 08:07:114639 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 15:53:224640 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4641
4642 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 15:44:424643 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 15:53:224644 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 08:07:114645 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 15:53:224646 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4647
Erik Språng9d69cbe2020-10-22 15:44:424648 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 15:53:224649 int number_of_layers_allocation = 1;
4650 const int64_t start_time_ms = CurrentTimeMs();
4651 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4652 video_source_.IncomingCapturedFrame(
4653 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4654 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 15:53:224655 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4656 number_of_layers_allocation = sink_.number_of_layers_allocations();
4657 VideoLayersAllocation new_allocation =
4658 sink_.GetLastVideoLayersAllocation();
4659 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4660 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4661 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4662 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4663 .target_bitrate_per_temporal_layer,
4664 last_layer_allocation.active_spatial_layers[0]
4665 .target_bitrate_per_temporal_layer);
4666 last_layer_allocation = new_allocation;
4667 }
4668 }
4669 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4670 video_stream_encoder_->Stop();
4671}
4672
4673TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134674 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354675 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4676 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4677 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354678 VideoEncoderConfig video_encoder_config;
4679 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4680 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114681 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354682 video_encoder_config.content_type =
4683 VideoEncoderConfig::ContentType::kRealtimeVideo;
4684 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434685 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354686 VideoEncoder::GetDefaultVp8Settings());
4687 for (auto& layer : video_encoder_config.simulcast_layers) {
4688 layer.num_temporal_layers = 2;
4689 }
4690 // Simulcast layers are used for enabling/disabling streams.
4691 video_encoder_config.simulcast_layers[0].active = true;
4692 video_encoder_config.simulcast_layers[1].active = false;
4693 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034694 ConfigureEncoder(std::move(video_encoder_config),
4695 VideoStreamEncoder::BitrateAllocationCallbackType::
4696 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354697
4698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354700
4701 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4702 WaitForEncodedFrame(CurrentTimeMs());
4703 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4704 VideoLayersAllocation last_layer_allocation =
4705 sink_.GetLastVideoLayersAllocation();
4706
4707 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4708 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4709 .target_bitrate_per_temporal_layer,
4710 SizeIs(2));
4711 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4712 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4713 video_stream_encoder_->Stop();
4714}
4715
4716TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134717 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354718 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4719 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4720 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354721 VideoEncoderConfig video_encoder_config;
4722 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4723 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114724 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354725 video_encoder_config.content_type =
4726 VideoEncoderConfig::ContentType::kRealtimeVideo;
4727 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434728 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354729 VideoEncoder::GetDefaultVp8Settings());
4730 for (auto& layer : video_encoder_config.simulcast_layers) {
4731 layer.num_temporal_layers = 2;
4732 }
4733 // Simulcast layers are used for enabling/disabling streams.
4734 video_encoder_config.simulcast_layers[0].active = true;
4735 video_encoder_config.simulcast_layers[1].active = false;
4736 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034737 ConfigureEncoder(std::move(video_encoder_config),
4738 VideoStreamEncoder::BitrateAllocationCallbackType::
4739 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354740
4741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114742 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354743
4744 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4745 WaitForEncodedFrame(CurrentTimeMs());
4746 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4747 VideoLayersAllocation last_layer_allocation =
4748 sink_.GetLastVideoLayersAllocation();
4749
4750 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4751 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4752 .target_bitrate_per_temporal_layer,
4753 SizeIs(2));
4754 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4755
4756 video_stream_encoder_->Stop();
4757}
4758
4759TEST_F(VideoStreamEncoderTest,
4760 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4761 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4762 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354763 VideoEncoderConfig video_encoder_config;
4764 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4765 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114766 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354767 video_encoder_config.content_type =
4768 VideoEncoderConfig::ContentType::kRealtimeVideo;
4769 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4770 vp9_settings.numberOfSpatialLayers = 2;
4771 vp9_settings.numberOfTemporalLayers = 2;
4772 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4773 vp9_settings.automaticResizeOn = false;
4774 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434775 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354776 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034777 ConfigureEncoder(std::move(video_encoder_config),
4778 VideoStreamEncoder::BitrateAllocationCallbackType::
4779 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354780
4781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114782 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354783
4784 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4785 WaitForEncodedFrame(CurrentTimeMs());
4786 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4787 VideoLayersAllocation last_layer_allocation =
4788 sink_.GetLastVideoLayersAllocation();
4789
4790 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4791 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4792 .target_bitrate_per_temporal_layer,
4793 SizeIs(2));
4794 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4795 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4796 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4797 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4798 .target_bitrate_per_temporal_layer,
4799 SizeIs(2));
4800 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4801 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4802 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4803
4804 // Since full SVC is used, expect the top layer to utilize the full target
4805 // rate.
4806 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4807 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114808 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354809 video_stream_encoder_->Stop();
4810}
4811
4812TEST_F(VideoStreamEncoderTest,
4813 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4814 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4815 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 14:27:354816 VideoEncoderConfig video_encoder_config;
4817 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4818 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114819 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354820 video_encoder_config.content_type =
4821 VideoEncoderConfig::ContentType::kRealtimeVideo;
4822 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4823 vp9_settings.numberOfSpatialLayers = 2;
4824 vp9_settings.numberOfTemporalLayers = 2;
4825 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4826 vp9_settings.automaticResizeOn = false;
4827 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434828 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354829 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034830 ConfigureEncoder(std::move(video_encoder_config),
4831 VideoStreamEncoder::BitrateAllocationCallbackType::
4832 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354833
4834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114835 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354836
4837 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4838 WaitForEncodedFrame(CurrentTimeMs());
4839 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4840 VideoLayersAllocation last_layer_allocation =
4841 sink_.GetLastVideoLayersAllocation();
4842
4843 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4844 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4845 .target_bitrate_per_temporal_layer,
4846 SizeIs(1));
4847 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4848 .target_bitrate_per_temporal_layer,
4849 SizeIs(1));
4850 // Since full SVC is used, expect the top layer to utilize the full target
4851 // rate.
4852 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4853 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:114854 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354855 video_stream_encoder_->Stop();
4856}
4857
4858TEST_F(VideoStreamEncoderTest,
4859 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4860 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4861 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354862 VideoEncoderConfig video_encoder_config;
4863 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4864 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114865 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354866 video_encoder_config.content_type =
4867 VideoEncoderConfig::ContentType::kRealtimeVideo;
4868 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4869 vp9_settings.numberOfSpatialLayers = 2;
4870 vp9_settings.numberOfTemporalLayers = 2;
4871 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4872 vp9_settings.automaticResizeOn = false;
4873 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434874 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354875 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034876 ConfigureEncoder(std::move(video_encoder_config),
4877 VideoStreamEncoder::BitrateAllocationCallbackType::
4878 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354879
4880 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114881 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354882
4883 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4884 WaitForEncodedFrame(CurrentTimeMs());
4885 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4886 VideoLayersAllocation last_layer_allocation =
4887 sink_.GetLastVideoLayersAllocation();
4888
4889 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4890 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4891 .target_bitrate_per_temporal_layer,
4892 SizeIs(2));
4893 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4894 .target_bitrate_per_temporal_layer,
4895 SizeIs(2));
4896 // Since KSVC is, spatial layers are independend except on key frames.
4897 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4898 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114899 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354900 video_stream_encoder_->Stop();
4901}
4902
4903TEST_F(VideoStreamEncoderTest,
4904 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4905 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4906 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4907 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354908 VideoEncoderConfig video_encoder_config;
4909 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4910 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114911 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354912 video_encoder_config.content_type =
4913 VideoEncoderConfig::ContentType::kRealtimeVideo;
4914 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4915 vp9_settings.numberOfSpatialLayers = 3;
4916 vp9_settings.numberOfTemporalLayers = 2;
4917 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4918 vp9_settings.automaticResizeOn = false;
4919 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434920 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354921 vp9_settings);
4922 // Simulcast layers are used for enabling/disabling streams.
4923 video_encoder_config.simulcast_layers.resize(3);
4924 video_encoder_config.simulcast_layers[0].active = false;
4925 video_encoder_config.simulcast_layers[1].active = true;
4926 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034927 ConfigureEncoder(std::move(video_encoder_config),
4928 VideoStreamEncoder::BitrateAllocationCallbackType::
4929 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354930
4931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114932 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354933
4934 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4935 WaitForEncodedFrame(CurrentTimeMs());
4936 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4937 VideoLayersAllocation last_layer_allocation =
4938 sink_.GetLastVideoLayersAllocation();
4939
4940 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4941 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4942 .target_bitrate_per_temporal_layer,
4943 SizeIs(2));
4944 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4945 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4946
4947 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4948 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4949 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4950 .target_bitrate_per_temporal_layer,
4951 SizeIs(2));
4952 // Since full SVC is used, expect the top layer to utilize the full target
4953 // rate.
4954 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4955 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:114956 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:354957 video_stream_encoder_->Stop();
4958}
4959
4960TEST_F(VideoStreamEncoderTest,
4961 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4962 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4963 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4964 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354965 VideoEncoderConfig video_encoder_config;
4966 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4967 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:114968 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:354969 video_encoder_config.content_type =
4970 VideoEncoderConfig::ContentType::kRealtimeVideo;
4971 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4972 vp9_settings.numberOfSpatialLayers = 3;
4973 vp9_settings.numberOfTemporalLayers = 2;
4974 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4975 vp9_settings.automaticResizeOn = false;
4976 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434977 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354978 vp9_settings);
4979 // Simulcast layers are used for enabling/disabling streams.
4980 video_encoder_config.simulcast_layers.resize(3);
4981 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034982 ConfigureEncoder(std::move(video_encoder_config),
4983 VideoStreamEncoder::BitrateAllocationCallbackType::
4984 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354985
4986 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:114987 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:354988
4989 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4990 WaitForEncodedFrame(CurrentTimeMs());
4991 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4992 VideoLayersAllocation last_layer_allocation =
4993 sink_.GetLastVideoLayersAllocation();
4994
4995 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4996 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4997 .target_bitrate_per_temporal_layer,
4998 SizeIs(2));
4999 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5000 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5001
5002 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5003 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5004 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5005 .target_bitrate_per_temporal_layer,
5006 SizeIs(2));
5007 video_stream_encoder_->Stop();
5008}
5009
5010TEST_F(VideoStreamEncoderTest,
5011 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5012 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5013 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5014 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:355015 VideoEncoderConfig video_encoder_config;
5016 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5017 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 08:07:115018 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 14:27:355019 video_encoder_config.content_type =
5020 VideoEncoderConfig::ContentType::kRealtimeVideo;
5021 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5022 vp9_settings.numberOfSpatialLayers = 3;
5023 vp9_settings.numberOfTemporalLayers = 2;
5024 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5025 vp9_settings.automaticResizeOn = false;
5026 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435027 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:355028 vp9_settings);
5029 // Simulcast layers are used for enabling/disabling streams.
5030 video_encoder_config.simulcast_layers.resize(3);
5031 video_encoder_config.simulcast_layers[0].active = false;
5032 video_encoder_config.simulcast_layers[1].active = false;
5033 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:035034 ConfigureEncoder(std::move(video_encoder_config),
5035 VideoStreamEncoder::BitrateAllocationCallbackType::
5036 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:355037
5038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115039 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:355040
5041 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5042 WaitForEncodedFrame(CurrentTimeMs());
5043 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5044 VideoLayersAllocation last_layer_allocation =
5045 sink_.GetLastVideoLayersAllocation();
5046
5047 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5048 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5049 .target_bitrate_per_temporal_layer,
5050 SizeIs(2));
5051 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5052 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5053 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5054 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 08:07:115055 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:355056 video_stream_encoder_->Stop();
5057}
5058
5059TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5060 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:035061 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 14:27:355062 kVideoLayersAllocation);
5063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115064 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 14:27:355065
5066 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5067 WaitForEncodedFrame(CurrentTimeMs());
5068 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5069 VideoLayersAllocation last_layer_allocation =
5070 sink_.GetLastVideoLayersAllocation();
5071
5072 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5073 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5074 .target_bitrate_per_temporal_layer,
5075 SizeIs(1));
5076 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5077 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:115078 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 14:27:355079 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5080 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5081 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5082 video_stream_encoder_->Stop();
5083}
5084
5085TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 15:53:225086 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5087 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035088 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:225089 kVideoLayersAllocation);
5090
5091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115092 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:225093
5094 video_source_.IncomingCapturedFrame(
5095 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5096 WaitForEncodedFrame(CurrentTimeMs());
5097 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5098 VideoLayersAllocation last_layer_allocation =
5099 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 08:07:115100 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 15:53:225101 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5102 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5103 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 08:07:115104 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 15:53:225105
5106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115107 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5108 0, 0, 0);
Per Kjellandera9434842020-10-15 15:53:225109 video_source_.IncomingCapturedFrame(
5110 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5111 WaitForEncodedFrame(CurrentTimeMs());
5112
5113 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5114 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5115 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5116 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5117 .target_bitrate_per_temporal_layer[0],
5118 DataRate::Zero());
5119
5120 video_stream_encoder_->Stop();
5121}
5122
Per Kjellander4190ce92020-12-15 16:24:555123TEST_F(VideoStreamEncoderTest,
5124 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5125 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035126 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 16:24:555127 kVideoLayersAllocation);
5128
5129 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115130 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5131 0, 0, 0);
Per Kjellander4190ce92020-12-15 16:24:555132
5133 video_source_.IncomingCapturedFrame(
5134 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5135 WaitForEncodedFrame(CurrentTimeMs());
5136 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5137 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5138 SizeIs(2));
5139 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5140 codec_width_);
5141 EXPECT_EQ(
5142 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5143 codec_height_);
5144
5145 video_source_.IncomingCapturedFrame(
5146 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5147 WaitForEncodedFrame(CurrentTimeMs());
5148 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5149 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5150 SizeIs(2));
5151 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5152 codec_width_ / 2);
5153 EXPECT_EQ(
5154 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5155 codec_height_ / 2);
5156
5157 video_stream_encoder_->Stop();
5158}
5159
Åsa Perssonc29cb2c2019-03-25 11:06:595160TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5161 // 2 TLs configured, temporal layers supported by encoder.
5162 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 13:09:055163 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035164 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055165 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595166 fake_encoder_.SetTemporalLayersSupported(0, true);
5167
5168 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 08:07:115169 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 11:06:595170 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:465171 kNumTemporalLayers, /*temporal_id*/ 0,
5172 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 08:07:115173 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 11:06:595174 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:465175 kNumTemporalLayers, /*temporal_id*/ 1,
5176 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595177 VideoBitrateAllocation expected_bitrate;
5178 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5179 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5180
5181 VerifyAllocatedBitrate(expected_bitrate);
5182 video_stream_encoder_->Stop();
5183}
5184
5185TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5186 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 13:09:055187 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035188 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055189 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595190 fake_encoder_.SetTemporalLayersSupported(0, false);
5191
5192 // Temporal layers not supported by the encoder.
5193 // Total bitrate should be at ti:0.
5194 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 08:07:115195 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 11:06:595196
5197 VerifyAllocatedBitrate(expected_bitrate);
5198 video_stream_encoder_->Stop();
5199}
5200
5201TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 13:09:055202 webrtc::test::ScopedFieldTrials field_trials(
5203 "WebRTC-Video-QualityScalerSettings/"
5204 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5205 // Reset encoder for field trials to take effect.
5206 ConfigureEncoder(video_encoder_config_.Copy());
5207
Åsa Perssonc29cb2c2019-03-25 11:06:595208 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 13:09:055209 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035210 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055211 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595212 fake_encoder_.SetTemporalLayersSupported(0, true);
5213 fake_encoder_.SetTemporalLayersSupported(1, false);
5214
5215 const int kS0Bps = 150000;
5216 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:465217 kS0Bps *
5218 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5219 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595220 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:465221 kS0Bps *
5222 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5223 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 08:07:115224 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 11:06:595225 // Temporal layers not supported by si:1.
5226 VideoBitrateAllocation expected_bitrate;
5227 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5228 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5229 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5230
5231 VerifyAllocatedBitrate(expected_bitrate);
5232 video_stream_encoder_->Stop();
5233}
5234
Niels Möller7dc26b72017-12-06 09:27:485235TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5236 const int kFrameWidth = 1280;
5237 const int kFrameHeight = 720;
5238 const int kFramerate = 24;
5239
Henrik Boström381d1092020-05-12 16:49:075240 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115241 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485242 test::FrameForwarder source;
5243 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415244 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485245
5246 // Insert a single frame, triggering initial configuration.
5247 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5248 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5249
5250 EXPECT_EQ(
5251 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5252 kDefaultFramerate);
5253
5254 // Trigger reconfigure encoder (without resetting the entire instance).
5255 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515256 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5257 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:115258 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 09:27:485259 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475260 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485261 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5262
5263 // Detector should be updated with fps limit from codec config.
5264 EXPECT_EQ(
5265 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5266 kFramerate);
5267
5268 // Trigger overuse, max framerate should be reduced.
5269 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5270 stats.input_frame_rate = kFramerate;
5271 stats_proxy_->SetMockStats(stats);
5272 video_stream_encoder_->TriggerCpuOveruse();
5273 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5274 int adapted_framerate =
5275 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5276 EXPECT_LT(adapted_framerate, kFramerate);
5277
5278 // Trigger underuse, max framerate should go back to codec configured fps.
5279 // Set extra low fps, to make sure it's actually reset, not just incremented.
5280 stats = stats_proxy_->GetStats();
5281 stats.input_frame_rate = adapted_framerate / 2;
5282 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335283 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485284 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5285 EXPECT_EQ(
5286 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5287 kFramerate);
5288
5289 video_stream_encoder_->Stop();
5290}
5291
5292TEST_F(VideoStreamEncoderTest,
5293 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5294 const int kFrameWidth = 1280;
5295 const int kFrameHeight = 720;
5296 const int kLowFramerate = 15;
5297 const int kHighFramerate = 25;
5298
Henrik Boström381d1092020-05-12 16:49:075299 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115300 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485301 test::FrameForwarder source;
5302 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415303 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485304
5305 // Trigger initial configuration.
5306 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515307 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5308 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 08:07:115309 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 09:27:485310 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 06:57:515311 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 07:51:475312 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5314
5315 EXPECT_EQ(
5316 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5317 kLowFramerate);
5318
5319 // Trigger overuse, max framerate should be reduced.
5320 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5321 stats.input_frame_rate = kLowFramerate;
5322 stats_proxy_->SetMockStats(stats);
5323 video_stream_encoder_->TriggerCpuOveruse();
5324 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5325 int adapted_framerate =
5326 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5327 EXPECT_LT(adapted_framerate, kLowFramerate);
5328
5329 // Reconfigure the encoder with a new (higher max framerate), max fps should
5330 // still respect the adaptation.
Åsa Persson17107062020-10-08 06:57:515331 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 09:27:485332 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5333 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475334 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485335 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5336
5337 EXPECT_EQ(
5338 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5339 adapted_framerate);
5340
5341 // Trigger underuse, max framerate should go back to codec configured fps.
5342 stats = stats_proxy_->GetStats();
5343 stats.input_frame_rate = adapted_framerate;
5344 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335345 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485346 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5347 EXPECT_EQ(
5348 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5349 kHighFramerate);
5350
5351 video_stream_encoder_->Stop();
5352}
5353
mflodmancc3d4422017-08-03 15:27:515354TEST_F(VideoStreamEncoderTest,
5355 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 11:21:075356 const int kFrameWidth = 1280;
5357 const int kFrameHeight = 720;
5358 const int kFramerate = 24;
5359
Henrik Boström381d1092020-05-12 16:49:075360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115361 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 11:21:075362 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:515363 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415364 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 11:21:075365
5366 // Trigger initial configuration.
5367 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515368 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5369 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:115370 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 11:21:075371 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 15:27:515372 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475373 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:515374 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 11:21:075375
Niels Möller7dc26b72017-12-06 09:27:485376 EXPECT_EQ(
5377 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5378 kFramerate);
sprangfda496a2017-06-15 11:21:075379
5380 // Trigger overuse, max framerate should be reduced.
5381 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5382 stats.input_frame_rate = kFramerate;
5383 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 15:27:515384 video_stream_encoder_->TriggerCpuOveruse();
5385 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:485386 int adapted_framerate =
5387 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 11:21:075388 EXPECT_LT(adapted_framerate, kFramerate);
5389
5390 // Change degradation preference to not enable framerate scaling. Target
5391 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 16:49:075392 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:415393 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 09:27:485394 EXPECT_EQ(
5395 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5396 kFramerate);
sprangfda496a2017-06-15 11:21:075397
mflodmancc3d4422017-08-03 15:27:515398 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 11:21:075399}
5400
mflodmancc3d4422017-08-03 15:27:515401TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495402 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075403 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075404 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5405 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5406 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495407 const int kWidth = 640;
5408 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225409
asaperssonfab67072017-04-04 12:51:495410 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225411
5412 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525413 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225414
5415 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095416 EXPECT_TRUE_WAIT(
5417 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 15:02:225418
sprangc5d62e22017-04-03 06:53:045419 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 15:02:225420
asaperssonfab67072017-04-04 12:51:495421 // Next frame is scaled.
kthelgason2bc68642017-02-07 15:02:225422 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:495423 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 15:02:225424
5425 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525426 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225427
Henrik Boström2671dac2020-05-19 14:29:095428 EXPECT_TRUE_WAIT(
5429 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 15:02:225430
mflodmancc3d4422017-08-03 15:27:515431 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225432}
5433
mflodmancc3d4422017-08-03 15:27:515434TEST_F(VideoStreamEncoderTest,
5435 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495436 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075438 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5439 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5440 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495441 const int kWidth = 640;
5442 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225443
5444 // We expect the n initial frames to get dropped.
5445 int i;
5446 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 12:51:495447 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525448 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225449 }
5450 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 12:51:495451 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525452 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 15:02:225453
5454 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 12:51:495455 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:225456
mflodmancc3d4422017-08-03 15:27:515457 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225458}
5459
mflodmancc3d4422017-08-03 15:27:515460TEST_F(VideoStreamEncoderTest,
5461 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 12:51:495462 const int kWidth = 640;
5463 const int kHeight = 360;
Henrik Boström381d1092020-05-12 16:49:075464 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115465 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 15:02:225466
5467 // Set degradation preference.
mflodmancc3d4422017-08-03 15:27:515468 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415469 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 15:02:225470
asaperssonfab67072017-04-04 12:51:495471 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225472 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525473 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 15:02:225474
mflodmancc3d4422017-08-03 15:27:515475 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225476}
5477
mflodmancc3d4422017-08-03 15:27:515478TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 12:51:495479 const int kWidth = 640;
5480 const int kHeight = 360;
kthelgasonad9010c2017-02-14 08:46:515481 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:135482
5483 VideoEncoderConfig video_encoder_config;
5484 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5485 // Make format different, to force recreation of encoder.
5486 video_encoder_config.video_format.parameters["foo"] = "foo";
5487 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475488 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 16:49:075489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115490 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-16 06:40:185491
kthelgasonb83797b2017-02-14 19:57:255492 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 21:20:415493 video_stream_encoder_->SetSource(&video_source_,
5494 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 08:46:515495
asaperssonfab67072017-04-04 12:51:495496 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 08:46:515497 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525498 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 08:46:515499
mflodmancc3d4422017-08-03 15:27:515500 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 08:46:515501 fake_encoder_.SetQualityScaling(true);
5502}
5503
Åsa Persson139f4dc2019-08-02 07:29:585504TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5505 webrtc::test::ScopedFieldTrials field_trials(
5506 "WebRTC-Video-QualityScalerSettings/"
5507 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5508 // Reset encoder for field trials to take effect.
5509 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:115510 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5511 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 07:29:585512 const int kWidth = 640;
5513 const int kHeight = 360;
5514
Henrik Boström381d1092020-05-12 16:49:075515 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115516 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585517 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5518 // Frame should not be dropped.
5519 WaitForEncodedFrame(1);
5520
Henrik Boström381d1092020-05-12 16:49:075521 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075522 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5523 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5524 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585525 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5526 // Frame should not be dropped.
5527 WaitForEncodedFrame(2);
5528
Henrik Boström381d1092020-05-12 16:49:075529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075530 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5531 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5532 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585533 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5534 // Expect to drop this frame, the wait should time out.
5535 ExpectDroppedFrame();
5536
5537 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095538 EXPECT_TRUE_WAIT(
5539 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 07:29:585540 video_stream_encoder_->Stop();
5541}
5542
Evan Shrubsolee3da1d32020-08-14 13:58:335543TEST_F(VideoStreamEncoderTest,
5544 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5545 webrtc::test::ScopedFieldTrials field_trials(
5546 "WebRTC-Video-QualityScalerSettings/"
5547 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5548 fake_encoder_.SetQualityScaling(false);
5549 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 08:07:115550 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5551 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 13:58:335552 const int kWidth = 640;
5553 const int kHeight = 360;
5554
5555 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115556 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 13:58:335557 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5558 // Frame should not be dropped.
5559 WaitForEncodedFrame(1);
5560
5561 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5562 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5563 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5564 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5565 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5566 // Frame should not be dropped.
5567 WaitForEncodedFrame(2);
5568
5569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5570 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5571 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5572 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5573 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5574 // Not dropped since quality scaling is disabled.
5575 WaitForEncodedFrame(3);
5576
5577 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 11:12:125578 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 13:58:335579 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5580
5581 video_stream_encoder_->Stop();
5582}
5583
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475584TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 08:07:115585 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475586 // Set simulcast.
5587 ResetEncoder("VP8", 3, 1, 1, false);
5588 fake_encoder_.SetQualityScaling(true);
5589 const int kWidth = 1280;
5590 const int kHeight = 720;
5591 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115592 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475593 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5594 // Frame should not be dropped.
5595 WaitForEncodedFrame(1);
5596
5597 // Trigger QVGA "singlecast"
5598 // Update the config.
5599 VideoEncoderConfig video_encoder_config;
5600 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5601 &video_encoder_config);
Åsa Persson7f354f82021-02-04 14:52:155602 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:435603 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 14:52:155604 "VP8", /*max qp*/ 56, /*screencast*/ false,
5605 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475606 for (auto& layer : video_encoder_config.simulcast_layers) {
5607 layer.num_temporal_layers = 1;
5608 layer.max_framerate = kDefaultFramerate;
5609 }
Asa Persson606d3cb2021-10-04 08:07:115610 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475611 video_encoder_config.content_type =
5612 VideoEncoderConfig::ContentType::kRealtimeVideo;
5613
5614 video_encoder_config.simulcast_layers[0].active = true;
5615 video_encoder_config.simulcast_layers[1].active = false;
5616 video_encoder_config.simulcast_layers[2].active = false;
5617
5618 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5619 kMaxPayloadLength);
5620 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5621
5622 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5623 // Frame should not be dropped.
5624 WaitForEncodedFrame(2);
5625
5626 // Trigger HD "singlecast"
5627 video_encoder_config.simulcast_layers[0].active = false;
5628 video_encoder_config.simulcast_layers[1].active = false;
5629 video_encoder_config.simulcast_layers[2].active = true;
5630
5631 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5632 kMaxPayloadLength);
5633 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5634
5635 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5636 // Frame should be dropped because of initial frame drop.
5637 ExpectDroppedFrame();
5638
5639 // Expect the sink_wants to specify a scaled frame.
5640 EXPECT_TRUE_WAIT(
5641 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5642 video_stream_encoder_->Stop();
5643}
5644
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085645TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 08:07:115646 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085647 // Set simulcast.
5648 ResetEncoder("VP9", 1, 1, 3, false);
5649 fake_encoder_.SetQualityScaling(true);
5650 const int kWidth = 1280;
5651 const int kHeight = 720;
5652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115653 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085654 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5655 // Frame should not be dropped.
5656 WaitForEncodedFrame(1);
5657
5658 // Trigger QVGA "singlecast"
5659 // Update the config.
5660 VideoEncoderConfig video_encoder_config;
5661 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5662 &video_encoder_config);
5663 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5664 vp9_settings.numberOfSpatialLayers = 3;
5665 // Since only one layer is active - automatic resize should be enabled.
5666 vp9_settings.automaticResizeOn = true;
5667 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435668 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085669 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115670 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085671 video_encoder_config.content_type =
5672 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 14:22:115673 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085674 // which SVC layers are active.
5675 video_encoder_config.simulcast_layers.resize(3);
5676
5677 video_encoder_config.simulcast_layers[0].active = true;
5678 video_encoder_config.simulcast_layers[1].active = false;
5679 video_encoder_config.simulcast_layers[2].active = false;
5680
5681 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5682 kMaxPayloadLength);
5683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5684
5685 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5686 // Frame should not be dropped.
5687 WaitForEncodedFrame(2);
5688
5689 // Trigger HD "singlecast"
5690 video_encoder_config.simulcast_layers[0].active = false;
5691 video_encoder_config.simulcast_layers[1].active = false;
5692 video_encoder_config.simulcast_layers[2].active = true;
5693
5694 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5695 kMaxPayloadLength);
5696 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5697
5698 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5699 // Frame should be dropped because of initial frame drop.
5700 ExpectDroppedFrame();
5701
5702 // Expect the sink_wants to specify a scaled frame.
5703 EXPECT_TRUE_WAIT(
5704 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5705 video_stream_encoder_->Stop();
5706}
5707
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475708TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055709 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5710 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5711 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5712 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5713 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5714 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5715 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5716 fake_encoder_.SetResolutionBitrateLimits(
5717 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5718
5719 VideoEncoderConfig video_encoder_config;
5720 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5721 &video_encoder_config);
5722 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5723 vp9_settings.numberOfSpatialLayers = 3;
5724 // Since only one layer is active - automatic resize should be enabled.
5725 vp9_settings.automaticResizeOn = true;
5726 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435727 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 08:20:055728 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115729 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 08:20:055730 video_encoder_config.content_type =
5731 VideoEncoderConfig::ContentType::kRealtimeVideo;
5732 // Simulcast layers are used to indicate which spatial layers are active.
5733 video_encoder_config.simulcast_layers.resize(3);
5734 video_encoder_config.simulcast_layers[0].active = false;
5735 video_encoder_config.simulcast_layers[1].active = true;
5736 video_encoder_config.simulcast_layers[2].active = false;
5737
5738 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5739 kMaxPayloadLength);
5740 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5741
5742 // The encoder bitrate limits for 360p should be used.
5743 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5744 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115745 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5746 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5747 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5748 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5749 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5750 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055751 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115752 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055753 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115754 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055755
5756 // The encoder bitrate limits for 270p should be used.
5757 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5758 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115759 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5760 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5761 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5762 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5763 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5764 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055765 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115766 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055767 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115768 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055769
5770 video_stream_encoder_->Stop();
5771}
5772
5773TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:515774 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5775 VideoEncoderConfig video_encoder_config;
5776 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5777 &video_encoder_config);
5778 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5779 vp9_settings.numberOfSpatialLayers = 3;
5780 // Since only one layer is active - automatic resize should be enabled.
5781 vp9_settings.automaticResizeOn = true;
5782 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435783 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 09:39:515784 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115785 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 09:39:515786 video_encoder_config.content_type =
5787 VideoEncoderConfig::ContentType::kRealtimeVideo;
5788 // Simulcast layers are used to indicate which spatial layers are active.
5789 video_encoder_config.simulcast_layers.resize(3);
5790 video_encoder_config.simulcast_layers[0].active = false;
5791 video_encoder_config.simulcast_layers[1].active = true;
5792 video_encoder_config.simulcast_layers[2].active = false;
5793
5794 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5795 kMaxPayloadLength);
5796 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5797
5798 // The default bitrate limits for 360p should be used.
5799 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195800 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5801 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515802 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5803 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115804 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5805 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5806 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5807 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5808 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5809 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515810 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115811 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515812 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115813 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515814
5815 // The default bitrate limits for 270p should be used.
5816 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 12:29:195817 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5818 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:515819 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5820 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115821 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5822 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5823 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5824 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5825 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5826 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515827 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115828 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515829 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115830 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515831
5832 video_stream_encoder_->Stop();
5833}
5834
5835TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5836 webrtc::test::ScopedFieldTrials field_trials(
5837 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5838 VideoEncoderConfig video_encoder_config;
5839 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5840 &video_encoder_config);
5841 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5842 vp9_settings.numberOfSpatialLayers = 3;
5843 // Since only one layer is active - automatic resize should be enabled.
5844 vp9_settings.automaticResizeOn = true;
5845 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435846 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 09:39:515847 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115848 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 09:39:515849 video_encoder_config.content_type =
5850 VideoEncoderConfig::ContentType::kRealtimeVideo;
5851 // Simulcast layers are used to indicate which spatial layers are active.
5852 video_encoder_config.simulcast_layers.resize(3);
5853 video_encoder_config.simulcast_layers[0].active = false;
5854 video_encoder_config.simulcast_layers[1].active = true;
5855 video_encoder_config.simulcast_layers[2].active = false;
5856
5857 // Reset encoder for field trials to take effect.
5858 ConfigureEncoder(video_encoder_config.Copy());
5859
5860 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5861 kMaxPayloadLength);
5862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5863
5864 // The default bitrate limits for 360p should not be used.
5865 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195866 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5867 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515868 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5869 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115870 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5871 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5872 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5873 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5874 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5875 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515876 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115877 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515878
5879 video_stream_encoder_->Stop();
5880}
5881
5882TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5883 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5884 /*num_spatial_layers=*/1, /*screenshare=*/false);
5885
5886 // The default singlecast bitrate limits for 720p should not be used.
5887 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 12:29:195888 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5889 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 09:39:515890 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5891 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115892 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5893 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5894 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5895 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5896 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5897 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 09:39:515898 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115899 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 09:39:515900
5901 video_stream_encoder_->Stop();
5902}
5903
5904TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055905 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5906 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5907 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5908 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5909 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5910 fake_encoder_.SetResolutionBitrateLimits(
5911 {kEncoderLimits180p, kEncoderLimits720p});
5912
5913 VideoEncoderConfig video_encoder_config;
5914 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5915 &video_encoder_config);
5916 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5917 vp9_settings.numberOfSpatialLayers = 3;
5918 // Since only one layer is active - automatic resize should be enabled.
5919 vp9_settings.automaticResizeOn = true;
5920 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435921 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 08:20:055922 vp9_settings);
Asa Persson606d3cb2021-10-04 08:07:115923 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 08:20:055924 video_encoder_config.content_type =
5925 VideoEncoderConfig::ContentType::kRealtimeVideo;
5926 // Simulcast layers are used to indicate which spatial layers are active.
5927 video_encoder_config.simulcast_layers.resize(3);
5928 video_encoder_config.simulcast_layers[0].active = true;
5929 video_encoder_config.simulcast_layers[1].active = false;
5930 video_encoder_config.simulcast_layers[2].active = false;
5931
5932 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5933 kMaxPayloadLength);
5934 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5935
5936 // Limits not applied on lowest stream, limits for 180p should not be used.
5937 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5938 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 08:07:115939 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5940 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5941 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5942 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5943 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5944 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 08:20:055945 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115946 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055947 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 08:07:115948 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 08:20:055949
5950 video_stream_encoder_->Stop();
5951}
5952
5953TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475954 InitialFrameDropActivatesWhenResolutionIncreases) {
5955 const int kWidth = 640;
5956 const int kHeight = 360;
5957
5958 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115959 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475960 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5961 // Frame should not be dropped.
5962 WaitForEncodedFrame(1);
5963
5964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115965 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475966 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5967 // Frame should not be dropped, bitrate not too low for frame.
5968 WaitForEncodedFrame(2);
5969
5970 // Incoming resolution increases.
5971 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5972 // Expect to drop this frame, bitrate too low for frame.
5973 ExpectDroppedFrame();
5974
5975 // Expect the sink_wants to specify a scaled frame.
5976 EXPECT_TRUE_WAIT(
5977 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5978 video_stream_encoder_->Stop();
5979}
5980
5981TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5982 const int kWidth = 640;
5983 const int kHeight = 360;
5984 // So that quality scaling doesn't happen by itself.
5985 fake_encoder_.SetQp(kQpHigh);
5986
5987 AdaptingFrameForwarder source(&time_controller_);
5988 source.set_adaptation_enabled(true);
5989 video_stream_encoder_->SetSource(
5990 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5991
5992 int timestamp = 1;
5993
5994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:115995 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475996 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5997 WaitForEncodedFrame(timestamp);
5998 timestamp += 9000;
5999 // Long pause to disable all first BWE drop logic.
6000 AdvanceTime(TimeDelta::Millis(1000));
6001
6002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116003 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:476004 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6005 // Not dropped frame, as initial frame drop is disabled by now.
6006 WaitForEncodedFrame(timestamp);
6007 timestamp += 9000;
6008 AdvanceTime(TimeDelta::Millis(100));
6009
6010 // Quality adaptation down.
6011 video_stream_encoder_->TriggerQualityLow();
6012
6013 // Adaptation has an effect.
6014 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6015 5000);
6016
6017 // Frame isn't dropped as initial frame dropper is disabled.
6018 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6019 WaitForEncodedFrame(timestamp);
6020 timestamp += 9000;
6021 AdvanceTime(TimeDelta::Millis(100));
6022
6023 // Quality adaptation up.
6024 video_stream_encoder_->TriggerQualityHigh();
6025
6026 // Adaptation has an effect.
6027 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6028 5000);
6029
6030 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6031 // Frame should not be dropped, as initial framedropper is off.
6032 WaitForEncodedFrame(timestamp);
6033
6034 video_stream_encoder_->Stop();
6035}
6036
Åsa Persson7f354f82021-02-04 14:52:156037TEST_F(VideoStreamEncoderTest,
6038 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6039 const int kMinStartBps360p = 222000;
6040 fake_encoder_.SetResolutionBitrateLimits(
6041 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6042 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6043 800000)});
6044
6045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6046 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6047 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6048 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6049 0, 0, 0);
6050 // Frame should not be dropped, bitrate not too low for frame.
6051 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6052 WaitForEncodedFrame(1);
6053
6054 // Incoming resolution increases, initial frame drop activates.
6055 // Frame should be dropped, link allocation too low for frame.
6056 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6057 ExpectDroppedFrame();
6058
6059 // Expect sink_wants to specify a scaled frame.
6060 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6061 5000);
6062 video_stream_encoder_->Stop();
6063}
6064
6065TEST_F(VideoStreamEncoderTest,
6066 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6067 const int kMinStartBps360p = 222000;
6068 fake_encoder_.SetResolutionBitrateLimits(
6069 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6070 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6071 800000)});
6072
6073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6074 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6075 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6076 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6077 0, 0, 0);
6078 // Frame should not be dropped, bitrate not too low for frame.
6079 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6080 WaitForEncodedFrame(1);
6081
6082 // Incoming resolution increases, initial frame drop activates.
6083 // Frame should be dropped, link allocation not too low for frame.
6084 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6085 WaitForEncodedFrame(2);
6086
6087 video_stream_encoder_->Stop();
6088}
6089
Åsa Perssone644a032019-11-08 14:56:006090TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6091 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 13:28:486092 "WebRTC-Video-QualityRampupSettings/"
6093 "min_pixels:921600,min_duration_ms:2000/");
6094
6095 const int kWidth = 1280;
6096 const int kHeight = 720;
6097 const int kFps = 10;
6098 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 14:56:006099
6100 // Reset encoder for field trials to take effect.
6101 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 08:07:116102 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 09:34:326103 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 14:56:006104 ConfigureEncoder(std::move(config));
6105 fake_encoder_.SetQp(kQpLow);
6106
6107 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 12:31:236108 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 14:56:006109 source.set_adaptation_enabled(true);
6110 video_stream_encoder_->SetSource(&source,
6111 DegradationPreference::MAINTAIN_FRAMERATE);
6112
6113 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 08:07:116114 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 16:49:076115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116116 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:006117
6118 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 13:28:486119 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 14:56:006120 int64_t timestamp_ms = kFrameIntervalMs;
6121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6122 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:096123 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6124 5000);
Åsa Perssone644a032019-11-08 14:56:006125
6126 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 16:49:076127 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6128 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:006129
Artem Titovab30d722021-07-27 14:22:116130 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 13:28:486131 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 14:56:006132 for (size_t i = 1; i <= 10; i++) {
6133 timestamp_ms += kFrameIntervalMs;
6134 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6135 WaitForEncodedFrame(timestamp_ms);
6136 }
Åsa Persson06defc42021-09-10 13:28:486137
6138 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6139 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6140 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6141 timestamp_ms += kFrameIntervalMs;
6142 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6143 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 14:56:006144 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6145 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6146
Åsa Persson06defc42021-09-10 13:28:486147 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 14:56:006148 timestamp_ms += kFrameIntervalMs;
6149 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6150 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 16:49:076151 // The ramp-up code involves the adaptation queue, give it time to execute.
6152 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 11:12:126153 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 12:08:396154 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 14:56:006155
6156 // Frame should not be adapted.
6157 timestamp_ms += kFrameIntervalMs;
6158 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6159 WaitForEncodedFrame(kWidth, kHeight);
6160 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6161
6162 video_stream_encoder_->Stop();
6163}
6164
mflodmancc3d4422017-08-03 15:27:516165TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 11:12:286166 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 16:19:316167 webrtc::test::ScopedFieldTrials field_trials(
6168 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 12:31:236169 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 11:12:286170 source.set_adaptation_enabled(true);
6171 video_stream_encoder_->SetSource(&source,
6172 DegradationPreference::MAINTAIN_FRAMERATE);
6173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116174 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 11:12:286175 fake_encoder_.SetQp(kQpHigh + 1);
6176 const int kWidth = 1280;
6177 const int kHeight = 720;
6178 const int64_t kFrameIntervalMs = 100;
6179 int64_t timestamp_ms = kFrameIntervalMs;
6180 for (size_t i = 1; i <= 100; i++) {
6181 timestamp_ms += kFrameIntervalMs;
6182 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6183 WaitForEncodedFrame(timestamp_ms);
6184 }
6185 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6186 // for the first time.
6187 // TODO(eshr): We should avoid these waits by using threads with simulated
6188 // time.
6189 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6190 2000 * 2.5 * 2);
6191 timestamp_ms += kFrameIntervalMs;
6192 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6193 WaitForEncodedFrame(timestamp_ms);
6194 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6195 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6196 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6197
6198 // Disable Quality scaling by turning off scaler on the encoder and
6199 // reconfiguring.
6200 fake_encoder_.SetQualityScaling(false);
6201 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6202 kMaxPayloadLength);
6203 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 09:11:556204 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 11:12:286205 // Since we turned off the quality scaler, the adaptations made by it are
6206 // removed.
6207 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6209
6210 video_stream_encoder_->Stop();
6211}
6212
6213TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:316214 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6215 const int kTooSmallWidth = 10;
6216 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 16:49:076217 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116218 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 08:47:316219
Taylor Brandstetter49fcc102018-05-16 21:20:416220 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:316221 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:516222 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416223 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 12:08:396224 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 08:47:316225 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6226
6227 // Trigger adapt down, too small frame, expect no change.
6228 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526229 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516230 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:396231 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:316232 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6233 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6234
mflodmancc3d4422017-08-03 15:27:516235 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:316236}
6237
mflodmancc3d4422017-08-03 15:27:516238TEST_F(VideoStreamEncoderTest,
6239 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226240 const int kTooSmallWidth = 10;
6241 const int kTooSmallHeight = 10;
6242 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 16:49:076243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116244 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226245
Taylor Brandstetter49fcc102018-05-16 21:20:416246 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:226247 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:416248 video_stream_encoder_->SetSource(&source,
6249 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:396250 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:226251 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6252 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6253
6254 // Trigger adapt down, expect limited framerate.
6255 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526256 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516257 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396258 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226259 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6260 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6261 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6262
6263 // Trigger adapt down, too small frame, expect no change.
6264 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526265 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:516266 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396267 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226268 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6269 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6270 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6271
mflodmancc3d4422017-08-03 15:27:516272 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226273}
6274
mflodmancc3d4422017-08-03 15:27:516275TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 08:12:526276 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 16:49:076277 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116278 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 07:51:476279 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 08:12:526280 const int kFrameWidth = 1280;
6281 const int kFrameHeight = 720;
6282 video_source_.IncomingCapturedFrame(
6283 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526284 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 15:27:516285 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:526286}
6287
sprangb1ca0732017-02-01 16:38:126288// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 15:27:516289TEST_F(VideoStreamEncoderTest,
6290 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 16:49:076291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116292 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 16:38:126293
6294 const int kFrameWidth = 1280;
6295 const int kFrameHeight = 720;
6296 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 15:27:516297 // requested by
6298 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 16:38:126299 video_source_.set_adaptation_enabled(true);
6300
6301 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196302 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526303 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126304
6305 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516306 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 16:38:126307 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196308 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526309 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 16:38:126310
asaperssonfab67072017-04-04 12:51:496311 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 10:24:336312 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 16:38:126313 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196314 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526315 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126316
mflodmancc3d4422017-08-03 15:27:516317 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 16:38:126318}
sprangfe627f32017-03-29 15:24:596319
mflodmancc3d4422017-08-03 15:27:516320TEST_F(VideoStreamEncoderTest,
6321 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-03 06:53:046322 const int kFrameWidth = 1280;
6323 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:046324
Henrik Boström381d1092020-05-12 16:49:076325 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116326 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516327 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416328 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046329 video_source_.set_adaptation_enabled(true);
6330
Tomas Gunnarsson612445e2020-09-21 12:31:236331 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046332
6333 video_source_.IncomingCapturedFrame(
6334 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526335 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046336
6337 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 15:27:516338 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046339
6340 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 14:06:526341 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046342 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046343 video_source_.IncomingCapturedFrame(
6344 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526345 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046346 }
6347
6348 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 15:27:516349 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046350 int num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526351 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046352 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046353 video_source_.IncomingCapturedFrame(
6354 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526355 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046356 ++num_frames_dropped;
6357 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566358 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046359 }
6360 }
6361
sprang4847ae62017-06-27 14:06:526362 // Add some slack to account for frames dropped by the frame dropper.
6363 const int kErrorMargin = 1;
6364 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046365 kErrorMargin);
6366
6367 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 15:27:516368 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046369 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 08:42:196370 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046371 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046372 video_source_.IncomingCapturedFrame(
6373 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526374 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046375 ++num_frames_dropped;
6376 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566377 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046378 }
6379 }
sprang4847ae62017-06-27 14:06:526380 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-03 06:53:046381 kErrorMargin);
6382
6383 // Go back up one step.
Henrik Boström91aa7322020-04-28 10:24:336384 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046385 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526386 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046387 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046388 video_source_.IncomingCapturedFrame(
6389 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526390 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046391 ++num_frames_dropped;
6392 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566393 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046394 }
6395 }
sprang4847ae62017-06-27 14:06:526396 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046397 kErrorMargin);
6398
6399 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 10:24:336400 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046401 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526402 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046403 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046404 video_source_.IncomingCapturedFrame(
6405 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526406 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046407 ++num_frames_dropped;
6408 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566409 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046410 }
6411 }
6412 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6413
mflodmancc3d4422017-08-03 15:27:516414 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046415}
6416
mflodmancc3d4422017-08-03 15:27:516417TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-03 06:53:046418 const int kFramerateFps = 5;
6419 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-03 06:53:046420 const int kFrameWidth = 1280;
6421 const int kFrameHeight = 720;
6422
sprang4847ae62017-06-27 14:06:526423 // Reconfigure encoder with two temporal layers and screensharing, which will
6424 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 07:51:476425 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 14:06:526426
Henrik Boström381d1092020-05-12 16:49:076427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116428 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516429 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416430 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046431 video_source_.set_adaptation_enabled(true);
6432
Tomas Gunnarsson612445e2020-09-21 12:31:236433 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046434
6435 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-09 01:04:296436 rtc::VideoSinkWants last_wants;
6437 do {
6438 last_wants = video_source_.sink_wants();
6439
sprangc5d62e22017-04-03 06:53:046440 // Insert frames to get a new fps estimate...
6441 for (int j = 0; j < kFramerateFps; ++j) {
6442 video_source_.IncomingCapturedFrame(
6443 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-09 01:04:296444 if (video_source_.last_sent_width()) {
6445 sink_.WaitForEncodedFrame(timestamp_ms);
6446 }
sprangc5d62e22017-04-03 06:53:046447 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:236448 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-03 06:53:046449 }
6450 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 15:27:516451 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:296452 } while (video_source_.sink_wants().max_framerate_fps <
6453 last_wants.max_framerate_fps);
sprangc5d62e22017-04-03 06:53:046454
Evan Shrubsole5cd7eb82020-05-25 12:08:396455 EXPECT_THAT(video_source_.sink_wants(),
6456 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-14 06:25:226457
mflodmancc3d4422017-08-03 15:27:516458 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046459}
asaperssonf7e294d2017-06-14 06:25:226460
mflodmancc3d4422017-08-03 15:27:516461TEST_F(VideoStreamEncoderTest,
6462 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226463 const int kWidth = 1280;
6464 const int kHeight = 720;
6465 const int64_t kFrameIntervalMs = 150;
6466 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116468 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226469
Taylor Brandstetter49fcc102018-05-16 21:20:416470 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236471 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226472 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416473 video_stream_encoder_->SetSource(&source,
6474 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226475 timestamp_ms += kFrameIntervalMs;
6476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526477 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396478 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226479 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6481 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6482
6483 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516484 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226485 timestamp_ms += kFrameIntervalMs;
6486 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526487 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396488 EXPECT_THAT(source.sink_wants(),
6489 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226490 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6491 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6492 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6493
6494 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516495 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226496 timestamp_ms += kFrameIntervalMs;
6497 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526498 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546499 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6502 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6503
6504 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516505 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226506 timestamp_ms += kFrameIntervalMs;
6507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526508 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546509 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226510 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6511 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6512 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6513
6514 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516515 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226516 timestamp_ms += kFrameIntervalMs;
6517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526518 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546519 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226520 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6521 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6522 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6523
6524 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516525 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226526 timestamp_ms += kFrameIntervalMs;
6527 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526528 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546529 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226530 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6531 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6532 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6533
6534 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516535 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226536 timestamp_ms += kFrameIntervalMs;
6537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526538 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546539 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226540 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6541 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6542 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6543
6544 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 15:27:516545 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226546 timestamp_ms += kFrameIntervalMs;
6547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526548 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546549 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226550 rtc::VideoSinkWants last_wants = source.sink_wants();
6551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6552 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6553 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6554
6555 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 15:27:516556 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226557 timestamp_ms += kFrameIntervalMs;
6558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526559 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546560 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-14 06:25:226561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6562 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6563 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6564
Åsa Perssonf5f7e8e2021-06-09 20:55:386565 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516566 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226567 timestamp_ms += kFrameIntervalMs;
6568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526569 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546570 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226571 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6573 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6574
6575 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516576 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226577 timestamp_ms += kFrameIntervalMs;
6578 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526579 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546580 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6583 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6584
6585 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516586 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226587 timestamp_ms += kFrameIntervalMs;
6588 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526589 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546590 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6593 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6594
6595 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516596 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226597 timestamp_ms += kFrameIntervalMs;
6598 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526599 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546600 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226601 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6602 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6603 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6604
6605 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516606 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226607 timestamp_ms += kFrameIntervalMs;
6608 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526609 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546610 EXPECT_THAT(source.sink_wants(), FpsMax());
6611 EXPECT_EQ(source.sink_wants().max_pixel_count,
6612 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-14 06:25:226613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6614 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6615 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6616
6617 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516618 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226619 timestamp_ms += kFrameIntervalMs;
6620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526621 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546622 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6624 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6625 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6626
Åsa Persson30ab0152019-08-27 10:22:336627 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516628 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226629 timestamp_ms += kFrameIntervalMs;
6630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526631 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546632 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396633 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226634 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6635 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6636 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6637
6638 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516639 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396640 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226641 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6642
mflodmancc3d4422017-08-03 15:27:516643 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226644}
6645
mflodmancc3d4422017-08-03 15:27:516646TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-14 06:25:226647 const int kWidth = 1280;
6648 const int kHeight = 720;
6649 const int64_t kFrameIntervalMs = 150;
6650 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076651 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116652 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226653
Taylor Brandstetter49fcc102018-05-16 21:20:416654 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236655 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226656 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416657 video_stream_encoder_->SetSource(&source,
6658 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226659 timestamp_ms += kFrameIntervalMs;
6660 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526661 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396662 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226663 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6664 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6665 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6666 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6667 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6668 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6669
6670 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516671 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226672 timestamp_ms += kFrameIntervalMs;
6673 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526674 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396675 EXPECT_THAT(source.sink_wants(),
6676 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226677 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6678 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6679 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6680 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6681 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6682 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6683
6684 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516685 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226686 timestamp_ms += kFrameIntervalMs;
6687 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526688 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546689 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226690 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6691 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6692 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6693 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6694 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6695 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6696
6697 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516698 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226699 timestamp_ms += kFrameIntervalMs;
6700 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526701 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546702 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 08:45:296703 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-14 06:25:226704 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6705 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6706 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6707 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6708 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6709
Evan Shrubsole64469032020-06-11 08:45:296710 // Trigger cpu adapt up, expect no change since QP is most limited.
6711 {
6712 // Store current sink wants since we expect no change and if there is no
6713 // change then last_wants() is not updated.
6714 auto previous_sink_wants = source.sink_wants();
6715 video_stream_encoder_->TriggerCpuUnderuse();
6716 timestamp_ms += kFrameIntervalMs;
6717 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6718 WaitForEncodedFrame(timestamp_ms);
6719 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6720 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6721 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6722 }
6723
6724 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6725 video_stream_encoder_->TriggerQualityHigh();
6726 timestamp_ms += kFrameIntervalMs;
6727 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6728 WaitForEncodedFrame(timestamp_ms);
6729 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6730 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6731 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6732 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6733 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6734 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6735 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6736
6737 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6738 // expect increased resolution (960x540@30fps).
6739 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336740 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226741 timestamp_ms += kFrameIntervalMs;
6742 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526743 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 08:45:296744 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226745 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6746 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6747 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6748 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6749 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296750 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226751
Evan Shrubsole64469032020-06-11 08:45:296752 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6753 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516754 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336755 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226756 timestamp_ms += kFrameIntervalMs;
6757 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526758 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546759 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396760 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226761 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6762 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6763 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6764 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6765 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296766 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226767
6768 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516769 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396770 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226771 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296772 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226773
mflodmancc3d4422017-08-03 15:27:516774 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226775}
6776
mflodmancc3d4422017-08-03 15:27:516777TEST_F(VideoStreamEncoderTest,
6778 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-14 06:25:226779 const int kWidth = 640;
6780 const int kHeight = 360;
6781 const int kFpsLimit = 15;
6782 const int64_t kFrameIntervalMs = 150;
6783 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116785 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226786
Taylor Brandstetter49fcc102018-05-16 21:20:416787 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236788 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226789 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416790 video_stream_encoder_->SetSource(&source,
6791 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226792 timestamp_ms += kFrameIntervalMs;
6793 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526794 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396795 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6798 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6800 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6801 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6802
6803 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516804 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226805 timestamp_ms += kFrameIntervalMs;
6806 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526807 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396808 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226809 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6810 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6811 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6812 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6813 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6814 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6815
6816 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516817 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226818 timestamp_ms += kFrameIntervalMs;
6819 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526820 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546821 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226822 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296823 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:226824 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6825 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6826 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6827 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6828
Evan Shrubsole64469032020-06-11 08:45:296829 // Trigger cpu adapt up, expect no change because quality is most limited.
6830 {
6831 auto previous_sink_wants = source.sink_wants();
6832 // Store current sink wants since we expect no change ind if there is no
6833 // change then last__wants() is not updated.
6834 video_stream_encoder_->TriggerCpuUnderuse();
6835 timestamp_ms += kFrameIntervalMs;
6836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6837 WaitForEncodedFrame(timestamp_ms);
6838 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6839 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6840 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6841 }
6842
6843 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6844 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226845 timestamp_ms += kFrameIntervalMs;
6846 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526847 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546848 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6850 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6851 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296852 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6853 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6854 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226855
Evan Shrubsole64469032020-06-11 08:45:296856 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516857 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 08:45:296858 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226859 timestamp_ms += kFrameIntervalMs;
6860 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526861 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396862 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226863 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6865 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6866 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6867 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296868 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226869
6870 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516871 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396872 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226873 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296874 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226875
mflodmancc3d4422017-08-03 15:27:516876 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226877}
6878
mflodmancc3d4422017-08-03 15:27:516879TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 13:53:486880 const int kFrameWidth = 1920;
6881 const int kFrameHeight = 1080;
6882 // 3/4 of 1920.
6883 const int kAdaptedFrameWidth = 1440;
6884 // 3/4 of 1080 rounded down to multiple of 4.
6885 const int kAdaptedFrameHeight = 808;
6886 const int kFramerate = 24;
6887
Henrik Boström381d1092020-05-12 16:49:076888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116889 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 13:53:486890 // Trigger reconfigure encoder (without resetting the entire instance).
6891 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 10:57:586892 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6893 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 08:07:116894 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 13:53:486895 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:436896 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 15:27:516897 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:476898 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:516899 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 13:53:486900
6901 video_source_.set_adaptation_enabled(true);
6902
6903 video_source_.IncomingCapturedFrame(
6904 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526905 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 13:53:486906
6907 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516908 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 13:53:486909 video_source_.IncomingCapturedFrame(
6910 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526911 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 13:53:486912
mflodmancc3d4422017-08-03 15:27:516913 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 13:53:486914}
6915
mflodmancc3d4422017-08-03 15:27:516916TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 14:06:526917 const int kFrameWidth = 1280;
6918 const int kFrameHeight = 720;
6919 const int kLowFps = 2;
6920 const int kHighFps = 30;
6921
Henrik Boström381d1092020-05-12 16:49:076922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116923 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:526924
Tomas Gunnarsson612445e2020-09-21 12:31:236925 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:526926 max_framerate_ = kLowFps;
6927
6928 // Insert 2 seconds of 2fps video.
6929 for (int i = 0; i < kLowFps * 2; ++i) {
6930 video_source_.IncomingCapturedFrame(
6931 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6932 WaitForEncodedFrame(timestamp_ms);
6933 timestamp_ms += 1000 / kLowFps;
6934 }
6935
6936 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 16:49:076937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116938 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 14:06:526939 video_source_.IncomingCapturedFrame(
6940 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6941 WaitForEncodedFrame(timestamp_ms);
6942 timestamp_ms += 1000 / kLowFps;
6943
6944 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6945
6946 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 08:48:486947 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 14:06:526948 const int kFrameIntervalMs = 1000 / kHighFps;
6949 max_framerate_ = kHighFps;
6950 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6951 video_source_.IncomingCapturedFrame(
6952 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6953 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6954 // be dropped if the encoder hans't been updated with the new higher target
6955 // framerate yet, causing it to overshoot the target bitrate and then
6956 // suffering the wrath of the media optimizer.
6957 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6958 timestamp_ms += kFrameIntervalMs;
6959 }
6960
6961 // Don expect correct measurement just yet, but it should be higher than
6962 // before.
6963 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6964
mflodmancc3d4422017-08-03 15:27:516965 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:526966}
6967
mflodmancc3d4422017-08-03 15:27:516968TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 14:06:526969 const int kFrameWidth = 1280;
6970 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 13:09:056971 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:036972 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:056973 kVideoBitrateAllocation);
sprang4847ae62017-06-27 14:06:526974
Henrik Boström381d1092020-05-12 16:49:076975 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116976 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516977 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 14:06:526978
6979 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 12:31:236980 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:526981 video_source_.IncomingCapturedFrame(
6982 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6983 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 13:09:056984 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:526985
6986 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 16:49:076987 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:116988 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 14:06:526989
6990 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 08:48:486991 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:236992 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 14:06:526993
Per Kjellanderdcef6412020-10-07 13:09:056994 // No more allocations has been made.
sprang4847ae62017-06-27 14:06:526995 video_source_.IncomingCapturedFrame(
6996 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6997 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 13:09:056998 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:526999
mflodmancc3d4422017-08-03 15:27:517000 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:527001}
ilnik6b826ef2017-06-16 13:53:487002
Niels Möller4db138e2018-04-19 07:04:137003TEST_F(VideoStreamEncoderTest,
7004 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7005 const int kFrameWidth = 1280;
7006 const int kFrameHeight = 720;
7007 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 16:49:077008 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117009 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:137010 video_source_.IncomingCapturedFrame(
7011 CreateFrame(1, kFrameWidth, kFrameHeight));
7012 WaitForEncodedFrame(1);
7013 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7014 .low_encode_usage_threshold_percent,
7015 default_options.low_encode_usage_threshold_percent);
7016 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7017 .high_encode_usage_threshold_percent,
7018 default_options.high_encode_usage_threshold_percent);
7019 video_stream_encoder_->Stop();
7020}
7021
7022TEST_F(VideoStreamEncoderTest,
7023 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7024 const int kFrameWidth = 1280;
7025 const int kFrameHeight = 720;
7026 CpuOveruseOptions hardware_options;
7027 hardware_options.low_encode_usage_threshold_percent = 150;
7028 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 11:42:187029 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 07:04:137030
Henrik Boström381d1092020-05-12 16:49:077031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117032 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:137033 video_source_.IncomingCapturedFrame(
7034 CreateFrame(1, kFrameWidth, kFrameHeight));
7035 WaitForEncodedFrame(1);
7036 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7037 .low_encode_usage_threshold_percent,
7038 hardware_options.low_encode_usage_threshold_percent);
7039 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7040 .high_encode_usage_threshold_percent,
7041 hardware_options.high_encode_usage_threshold_percent);
7042 video_stream_encoder_->Stop();
7043}
7044
Jakob Ivarsson461b1d92021-01-22 15:27:437045TEST_F(VideoStreamEncoderTest,
7046 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7047 const int kFrameWidth = 1280;
7048 const int kFrameHeight = 720;
7049
7050 const CpuOveruseOptions default_options;
7051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117052 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 15:27:437053 video_source_.IncomingCapturedFrame(
7054 CreateFrame(1, kFrameWidth, kFrameHeight));
7055 WaitForEncodedFrame(1);
7056 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7057 .low_encode_usage_threshold_percent,
7058 default_options.low_encode_usage_threshold_percent);
7059 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7060 .high_encode_usage_threshold_percent,
7061 default_options.high_encode_usage_threshold_percent);
7062
7063 CpuOveruseOptions hardware_options;
7064 hardware_options.low_encode_usage_threshold_percent = 150;
7065 hardware_options.high_encode_usage_threshold_percent = 200;
7066 fake_encoder_.SetIsHardwareAccelerated(true);
7067
7068 video_source_.IncomingCapturedFrame(
7069 CreateFrame(2, kFrameWidth, kFrameHeight));
7070 WaitForEncodedFrame(2);
7071
7072 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7073 .low_encode_usage_threshold_percent,
7074 hardware_options.low_encode_usage_threshold_percent);
7075 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7076 .high_encode_usage_threshold_percent,
7077 hardware_options.high_encode_usage_threshold_percent);
7078
7079 video_stream_encoder_->Stop();
7080}
7081
Niels Möller6bb5ab92019-01-11 10:11:107082TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7083 const int kFrameWidth = 320;
7084 const int kFrameHeight = 240;
7085 const int kFps = 30;
Asa Persson606d3cb2021-10-04 08:07:117086 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 10:11:107087 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7088
Henrik Boström381d1092020-05-12 16:49:077089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117090 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107091
Tomas Gunnarsson612445e2020-09-21 12:31:237092 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:107093 max_framerate_ = kFps;
7094
7095 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7096 fake_encoder_.SimulateOvershoot(1.0);
7097 int num_dropped = 0;
7098 for (int i = 0; i < kNumFramesInRun; ++i) {
7099 video_source_.IncomingCapturedFrame(
7100 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7101 // Wait up to two frame durations for a frame to arrive.
7102 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7103 ++num_dropped;
7104 }
7105 timestamp_ms += 1000 / kFps;
7106 }
7107
Erik Språnga8d48ab2019-02-08 13:17:407108 // Framerate should be measured to be near the expected target rate.
7109 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7110
7111 // Frame drops should be within 5% of expected 0%.
7112 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:107113
7114 // Make encoder produce frames at double the expected bitrate during 3 seconds
7115 // of video, verify number of drops. Rate needs to be slightly changed in
7116 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 15:20:177117 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 15:44:427118 const RateControlSettings trials =
7119 RateControlSettings::ParseFromFieldTrials();
7120 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 15:20:177121 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 15:44:427122 // frame dropping since the adjuter will try to just lower the target
7123 // bitrate rather than drop frames. If network headroom can be used, it
7124 // doesn't push back as hard so we don't need quite as much overshoot.
7125 // These numbers are unfortunately a bit magical but there's not trivial
7126 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 10:10:097127 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 15:20:177128 }
7129 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 16:49:077130 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117131 kTargetBitrate + DataRate::KilobitsPerSec(1),
7132 kTargetBitrate + DataRate::KilobitsPerSec(1),
7133 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107134 num_dropped = 0;
7135 for (int i = 0; i < kNumFramesInRun; ++i) {
7136 video_source_.IncomingCapturedFrame(
7137 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7138 // Wait up to two frame durations for a frame to arrive.
7139 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7140 ++num_dropped;
7141 }
7142 timestamp_ms += 1000 / kFps;
7143 }
7144
Henrik Boström381d1092020-05-12 16:49:077145 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117146 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 13:17:407147
7148 // Target framerate should be still be near the expected target, despite
7149 // the frame drops.
7150 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7151
7152 // Frame drops should be within 5% of expected 50%.
7153 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:107154
7155 video_stream_encoder_->Stop();
7156}
7157
7158TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7159 const int kFrameWidth = 320;
7160 const int kFrameHeight = 240;
7161 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 08:07:117162 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 10:11:107163
7164 ASSERT_GT(max_framerate_, kActualInputFps);
7165
Tomas Gunnarsson612445e2020-09-21 12:31:237166 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:107167 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 16:49:077168 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117169 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107170
7171 // Insert 3 seconds of video, with an input fps lower than configured max.
7172 for (int i = 0; i < kActualInputFps * 3; ++i) {
7173 video_source_.IncomingCapturedFrame(
7174 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7175 // Wait up to two frame durations for a frame to arrive.
7176 WaitForEncodedFrame(timestamp_ms);
7177 timestamp_ms += 1000 / kActualInputFps;
7178 }
7179
7180 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7181
7182 video_stream_encoder_->Stop();
7183}
7184
Markus Handell9a478b52021-11-18 15:07:017185TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267186 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 15:07:017187 test::FrameForwarder source;
7188 video_stream_encoder_->SetSource(&source,
7189 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 16:49:077190 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117191 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267192
Markus Handell9a478b52021-11-18 15:07:017193 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267194 WaitForEncodedFrame(1);
7195 // On the very first frame full update should be forced.
7196 rect = fake_encoder_.GetLastUpdateRect();
7197 EXPECT_EQ(rect.offset_x, 0);
7198 EXPECT_EQ(rect.offset_y, 0);
7199 EXPECT_EQ(rect.height, codec_height_);
7200 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 15:07:017201 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7202 // scheduled for processing during encoder queue processing of frame 2.
7203 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7204 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267205 WaitForEncodedFrame(3);
7206 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7207 rect = fake_encoder_.GetLastUpdateRect();
7208 EXPECT_EQ(rect.offset_x, 1);
7209 EXPECT_EQ(rect.offset_y, 0);
7210 EXPECT_EQ(rect.width, 10);
7211 EXPECT_EQ(rect.height, 1);
7212
Markus Handell9a478b52021-11-18 15:07:017213 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267214 WaitForEncodedFrame(4);
7215 // Previous frame was encoded, so no accumulation should happen.
7216 rect = fake_encoder_.GetLastUpdateRect();
7217 EXPECT_EQ(rect.offset_x, 0);
7218 EXPECT_EQ(rect.offset_y, 0);
7219 EXPECT_EQ(rect.width, 1);
7220 EXPECT_EQ(rect.height, 1);
7221
7222 video_stream_encoder_->Stop();
7223}
7224
Erik Språngd7329ca2019-02-21 20:19:537225TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 16:49:077226 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117227 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537228
7229 // First frame is always keyframe.
7230 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7231 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 14:43:587232 EXPECT_THAT(
7233 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127234 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537235
7236 // Insert delta frame.
7237 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7238 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 14:43:587239 EXPECT_THAT(
7240 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127241 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537242
7243 // Request next frame be a key-frame.
7244 video_stream_encoder_->SendKeyFrame();
7245 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7246 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 14:43:587247 EXPECT_THAT(
7248 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127249 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537250
7251 video_stream_encoder_->Stop();
7252}
7253
7254TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7255 // Setup simulcast with three streams.
7256 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077257 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117258 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7259 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537260 // Wait for all three layers before triggering event.
7261 sink_.SetNumExpectedLayers(3);
7262
7263 // First frame is always keyframe.
7264 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7265 WaitForEncodedFrame(1);
7266 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127267 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7268 VideoFrameType::kVideoFrameKey,
7269 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537270
7271 // Insert delta frame.
7272 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7273 WaitForEncodedFrame(2);
7274 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127275 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7276 VideoFrameType::kVideoFrameDelta,
7277 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537278
7279 // Request next frame be a key-frame.
7280 // Only first stream is configured to produce key-frame.
7281 video_stream_encoder_->SendKeyFrame();
7282 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7283 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 11:45:397284
7285 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7286 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 20:19:537287 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127288 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 11:45:397289 VideoFrameType::kVideoFrameKey,
7290 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537291
7292 video_stream_encoder_->Stop();
7293}
7294
Mirta Dvornicic28f0eb22019-05-28 14:30:167295TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237296 // SPS contains VUI with restrictions on the maximum number of reordered
7297 // pictures, there is no need to rewrite the bitstream to enable faster
7298 // decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167299 ResetEncoder("H264", 1, 1, 1, false);
7300
Mirta Dvornicic97910da2020-07-14 13:29:237301 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117302 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 13:29:237303 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167304
Mirta Dvornicic97910da2020-07-14 13:29:237305 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 08:07:117306 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167307
Mirta Dvornicic97910da2020-07-14 13:29:237308 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7309 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167310
7311 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 08:07:117312 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167313
7314 video_stream_encoder_->Stop();
7315}
7316
7317TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237318 // SPS does not contain VUI, the bitstream is will be rewritten with added
7319 // VUI with restrictions on the maximum number of reordered pictures to
7320 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167321 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7322 0x00, 0x00, 0x03, 0x03, 0xF4,
7323 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 14:30:167324 ResetEncoder("H264", 1, 1, 1, false);
7325
Mirta Dvornicic97910da2020-07-14 13:29:237326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117327 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 13:29:237328 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167329
Mirta Dvornicic97910da2020-07-14 13:29:237330 fake_encoder_.SetEncodedImageData(
7331 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167332
Mirta Dvornicic97910da2020-07-14 13:29:237333 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7334 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167335
7336 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 08:07:117337 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167338
7339 video_stream_encoder_->Stop();
7340}
7341
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147342TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7343 const int kFrameWidth = 1280;
7344 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 08:07:117345 const DataRate kTargetBitrate =
7346 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147347
Henrik Boström381d1092020-05-12 16:49:077348 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117349 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147350 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7351
7352 // Insert a first video frame. It should be dropped because of downscale in
7353 // resolution.
Tomas Gunnarsson612445e2020-09-21 12:31:237354 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147355 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7356 frame.set_rotation(kVideoRotation_270);
7357 video_source_.IncomingCapturedFrame(frame);
7358
7359 ExpectDroppedFrame();
7360
7361 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237362 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147363 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7364 frame.set_rotation(kVideoRotation_90);
7365 video_source_.IncomingCapturedFrame(frame);
7366
7367 WaitForEncodedFrame(timestamp_ms);
7368 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7369
7370 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237371 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147372 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7373 frame.set_rotation(kVideoRotation_180);
7374 video_source_.IncomingCapturedFrame(frame);
7375
7376 WaitForEncodedFrame(timestamp_ms);
7377 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7378
7379 video_stream_encoder_->Stop();
7380}
7381
Erik Språng5056af02019-09-02 13:53:117382TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7383 const int kFrameWidth = 320;
7384 const int kFrameHeight = 180;
7385
7386 // Initial rate.
Henrik Boström381d1092020-05-12 16:49:077387 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077388 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7389 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7390 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 13:53:117391 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487392 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327393 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117394
7395 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237396 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 13:53:117397 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7398 frame.set_rotation(kVideoRotation_270);
7399 video_source_.IncomingCapturedFrame(frame);
7400 WaitForEncodedFrame(timestamp_ms);
7401
7402 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 08:07:117403 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 17:46:077404 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7405 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 16:49:077406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 13:53:117407 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 11:36:557408 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 13:53:117409 /*link_allocation=*/target_rate,
7410 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487411 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327412 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117413 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7414
7415 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7416 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7417 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 17:46:077418 DataRate allocation_sum =
7419 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 13:53:117420 EXPECT_EQ(min_rate, allocation_sum);
7421 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7422
7423 video_stream_encoder_->Stop();
7424}
7425
Rasmus Brandt5cad55b2019-12-19 08:47:117426TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 16:49:077427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117428 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 10:50:237429 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 12:31:237430 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 10:50:237431 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7432 WaitForEncodedFrame(1);
7433
7434 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7435 ASSERT_TRUE(prev_rate_settings.has_value());
7436 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7437 kDefaultFramerate);
7438
7439 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7440 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7441 timestamp_ms += 1000 / kDefaultFramerate;
7442 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7443 WaitForEncodedFrame(timestamp_ms);
7444 }
7445 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7446 kDefaultFramerate);
7447 // Capture larger frame to trigger a reconfigure.
7448 codec_height_ *= 2;
7449 codec_width_ *= 2;
7450 timestamp_ms += 1000 / kDefaultFramerate;
7451 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7452 WaitForEncodedFrame(timestamp_ms);
7453
7454 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7455 auto current_rate_settings =
7456 fake_encoder_.GetAndResetLastRateControlSettings();
7457 // Ensure we have actually reconfigured twice
7458 // The rate settings should have been set again even though
7459 // they haven't changed.
7460 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 07:55:037461 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 10:50:237462
7463 video_stream_encoder_->Stop();
7464}
7465
philipeld9cc8c02019-09-16 12:53:407466struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 17:17:517467 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 17:17:517468 MOCK_METHOD(void,
7469 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 10:35:047470 (const webrtc::SdpVideoFormat& format,
7471 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 17:17:517472 (override));
philipeld9cc8c02019-09-16 12:53:407473};
7474
philipel9b058032020-02-10 10:30:007475TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7476 constexpr int kDontCare = 100;
7477 StrictMock<MockEncoderSelector> encoder_selector;
7478 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7479 &fake_encoder_, &encoder_selector);
7480 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7481
7482 // Reset encoder for new configuration to take effect.
7483 ConfigureEncoder(video_encoder_config_.Copy());
7484
Sergey Silkine1cd3ad2022-01-21 10:35:047485 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 10:30:007486
7487 video_source_.IncomingCapturedFrame(
7488 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 09:11:557489 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 10:30:007490 video_stream_encoder_->Stop();
7491
Sergey Silkine1cd3ad2022-01-21 10:35:047492 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 10:30:007493 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 14:22:117494 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7495 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 10:30:007496 video_stream_encoder_.reset();
7497}
7498
7499TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7500 constexpr int kDontCare = 100;
7501
7502 NiceMock<MockEncoderSelector> encoder_selector;
7503 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7504 video_send_config_.encoder_settings.encoder_switch_request_callback =
7505 &switch_callback;
7506 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7507 &fake_encoder_, &encoder_selector);
7508 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7509
7510 // Reset encoder for new configuration to take effect.
7511 ConfigureEncoder(video_encoder_config_.Copy());
7512
Sergey Silkine1cd3ad2022-01-21 10:35:047513 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 10:30:007514 .WillByDefault(Return(SdpVideoFormat("AV1")));
7515 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 10:35:047516 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7517 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 10:30:007518
Henrik Boström381d1092020-05-12 16:49:077519 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077520 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7521 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7522 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 10:30:007523 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487524 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 10:30:007525 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 09:11:557526 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 10:30:007527
7528 video_stream_encoder_->Stop();
7529}
7530
7531TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7532 constexpr int kSufficientBitrateToNotDrop = 1000;
7533 constexpr int kDontCare = 100;
7534
7535 NiceMock<MockVideoEncoder> video_encoder;
7536 NiceMock<MockEncoderSelector> encoder_selector;
7537 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7538 video_send_config_.encoder_settings.encoder_switch_request_callback =
7539 &switch_callback;
7540 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7541 &video_encoder, &encoder_selector);
7542 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7543
7544 // Reset encoder for new configuration to take effect.
7545 ConfigureEncoder(video_encoder_config_.Copy());
7546
7547 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7548 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7549 // not fail.
Henrik Boström381d1092020-05-12 16:49:077550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077551 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7552 /*stable_target_bitrate=*/
7553 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7554 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 10:30:007555 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487556 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 10:30:007557 /*cwnd_reduce_ratio=*/0);
7558
Sergey Silkine1cd3ad2022-01-21 10:35:047559 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 10:30:007560 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 10:35:047561 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 10:30:007562 .WillByDefault(Return(SdpVideoFormat("AV2")));
7563
7564 rtc::Event encode_attempted;
7565 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 10:35:047566 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7567 /*allow_default_fallback=*/true))
7568 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 10:30:007569
7570 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7571 encode_attempted.Wait(3000);
7572
Markus Handell28c71802021-11-08 09:11:557573 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 13:56:427574
philipel9b058032020-02-10 10:30:007575 video_stream_encoder_->Stop();
7576
Sergey Silkine1cd3ad2022-01-21 10:35:047577 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7578 // to it's factory, so in order for the encoder instance in the
7579 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7580 // reset the `video_stream_encoder_` here.
7581 video_stream_encoder_.reset();
7582}
7583
7584TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7585 NiceMock<MockVideoEncoder> video_encoder;
7586 NiceMock<MockEncoderSelector> encoder_selector;
7587 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7588 video_send_config_.encoder_settings.encoder_switch_request_callback =
7589 &switch_callback;
7590 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7591 &video_encoder, &encoder_selector);
7592 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7593
7594 // Reset encoder for new configuration to take effect.
7595 ConfigureEncoder(video_encoder_config_.Copy());
7596
7597 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7598 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7599 /*round_trip_time_ms=*/0,
7600 /*cwnd_reduce_ratio=*/0);
7601 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7602
7603 ON_CALL(video_encoder, InitEncode(_, _))
7604 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7605 ON_CALL(encoder_selector, OnEncoderBroken)
7606 .WillByDefault(Return(SdpVideoFormat("AV2")));
7607
7608 rtc::Event encode_attempted;
7609 EXPECT_CALL(switch_callback,
7610 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7611 /*allow_default_fallback=*/true))
7612 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7613
7614 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7615 encode_attempted.Wait(3000);
7616
7617 AdvanceTime(TimeDelta::Zero());
7618
7619 video_stream_encoder_->Stop();
7620
7621 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7622 // to it's factory, so in order for the encoder instance in the
7623 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7624 // reset the `video_stream_encoder_` here.
7625 video_stream_encoder_.reset();
7626}
7627
7628TEST_F(VideoStreamEncoderTest,
7629 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7630 NiceMock<MockVideoEncoder> video_encoder;
7631 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7632 video_send_config_.encoder_settings.encoder_switch_request_callback =
7633 &switch_callback;
7634 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7635 &video_encoder, /*encoder_selector=*/nullptr);
7636 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7637
7638 // Reset encoder for new configuration to take effect.
7639 ConfigureEncoder(video_encoder_config_.Copy());
7640
7641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7642 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7643 /*round_trip_time_ms=*/0,
7644 /*cwnd_reduce_ratio=*/0);
7645 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7646
7647 ON_CALL(video_encoder, InitEncode(_, _))
7648 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7649
7650 rtc::Event encode_attempted;
7651 EXPECT_CALL(switch_callback,
7652 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7653 /*allow_default_fallback=*/true))
7654 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7655
7656 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7657 encode_attempted.Wait(3000);
7658
7659 AdvanceTime(TimeDelta::Zero());
7660
7661 video_stream_encoder_->Stop();
7662
7663 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 10:30:007664 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 14:22:117665 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7666 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 10:30:007667 video_stream_encoder_.reset();
7668}
7669
Evan Shrubsole7c079f62019-09-26 07:55:037670TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117671 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037672 const int kFrameWidth = 320;
7673 const int kFrameHeight = 180;
7674
7675 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077676 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037678 /*target_bitrate=*/rate,
7679 /*stable_target_bitrate=*/rate,
7680 /*link_allocation=*/rate,
7681 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487682 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327683 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037684
7685 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237686 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037687 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7688 frame.set_rotation(kVideoRotation_270);
7689 video_source_.IncomingCapturedFrame(frame);
7690 WaitForEncodedFrame(timestamp_ms);
7691 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7692
7693 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 17:46:077694 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037696 /*target_bitrate=*/new_stable_rate,
7697 /*stable_target_bitrate=*/new_stable_rate,
7698 /*link_allocation=*/rate,
7699 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487700 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327701 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037702 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7703 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7704 video_stream_encoder_->Stop();
7705}
7706
7707TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117708 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037709 const int kFrameWidth = 320;
7710 const int kFrameHeight = 180;
7711
7712 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077713 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037715 /*target_bitrate=*/rate,
7716 /*stable_target_bitrate=*/rate,
7717 /*link_allocation=*/rate,
7718 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487719 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327720 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037721
7722 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237723 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037724 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7725 frame.set_rotation(kVideoRotation_270);
7726 video_source_.IncomingCapturedFrame(frame);
7727 WaitForEncodedFrame(timestamp_ms);
7728 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7729
7730 // Set a higher target rate without changing the link_allocation. Should not
7731 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077732 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037734 /*target_bitrate=*/rate,
7735 /*stable_target_bitrate=*/new_stable_rate,
7736 /*link_allocation=*/rate,
7737 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 14:14:487738 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 13:29:327739 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037740 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7741 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7742 video_stream_encoder_->Stop();
7743}
7744
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177745TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7746 test::ScopedFieldTrials field_trials(
7747 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7748 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7749 const int kFramerateFps = 30;
7750 const int kWidth = 1920;
7751 const int kHeight = 1080;
7752 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7753 // Works on screenshare mode.
7754 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7755 // We rely on the automatic resolution adaptation, but we handle framerate
7756 // adaptation manually by mocking the stats proxy.
7757 video_source_.set_adaptation_enabled(true);
7758
7759 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 16:49:077760 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117761 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177762 video_stream_encoder_->SetSource(&video_source_,
7763 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:397764 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177765
7766 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7767 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7768
7769 // Pass enough frames with the full update to trigger animation detection.
7770 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237771 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177772 frame.set_ntp_time_ms(timestamp_ms);
7773 frame.set_timestamp_us(timestamp_ms * 1000);
7774 video_source_.IncomingCapturedFrame(frame);
7775 WaitForEncodedFrame(timestamp_ms);
7776 }
7777
7778 // Resolution should be limited.
7779 rtc::VideoSinkWants expected;
7780 expected.max_framerate_fps = kFramerateFps;
7781 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 14:19:547782 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177783
7784 // Pass one frame with no known update.
7785 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 12:31:237786 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177787 frame.set_ntp_time_ms(timestamp_ms);
7788 frame.set_timestamp_us(timestamp_ms * 1000);
7789 frame.clear_update_rect();
7790
7791 video_source_.IncomingCapturedFrame(frame);
7792 WaitForEncodedFrame(timestamp_ms);
7793
7794 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 12:08:397795 EXPECT_THAT(video_source_.sink_wants(),
7796 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177797
7798 video_stream_encoder_->Stop();
7799}
7800
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327801TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7802 const int kWidth = 720; // 540p adapted down.
7803 const int kHeight = 405;
7804 const int kNumFrames = 3;
7805 // Works on screenshare mode.
7806 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7807 /*num_spatial_layers=*/2, /*screenshare=*/true);
7808
7809 video_source_.set_adaptation_enabled(true);
7810
7811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117812 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327813
7814 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7815
7816 // Pass enough frames with the full update to trigger animation detection.
7817 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237818 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327819 frame.set_ntp_time_ms(timestamp_ms);
7820 frame.set_timestamp_us(timestamp_ms * 1000);
7821 video_source_.IncomingCapturedFrame(frame);
7822 WaitForEncodedFrame(timestamp_ms);
7823 }
7824
7825 video_stream_encoder_->Stop();
7826}
7827
Yun Zhang1e4d4fd2020-09-30 07:22:467828TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7829 const float downscale_factors[] = {4.0, 2.0, 1.0};
7830 const int number_layers =
7831 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7832 VideoEncoderConfig config;
7833 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7834 for (int i = 0; i < number_layers; ++i) {
7835 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7836 config.simulcast_layers[i].active = true;
7837 }
7838 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:437839 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 07:22:467840 "VP8", /*max qp*/ 56, /*screencast*/ false,
7841 /*screenshare enabled*/ false);
7842 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117843 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7844 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 07:22:467845
7846 // First initialization.
7847 // Encoder should be initialized. Next frame should be key frame.
7848 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7849 sink_.SetNumExpectedLayers(number_layers);
7850 int64_t timestamp_ms = kFrameIntervalMs;
7851 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7852 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117853 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467854 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7855 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7856 VideoFrameType::kVideoFrameKey,
7857 VideoFrameType::kVideoFrameKey}));
7858
7859 // Disable top layer.
7860 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7861 config.simulcast_layers[number_layers - 1].active = false;
7862 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7863 sink_.SetNumExpectedLayers(number_layers - 1);
7864 timestamp_ms += kFrameIntervalMs;
7865 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7866 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117867 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467868 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7869 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7870 VideoFrameType::kVideoFrameDelta,
7871 VideoFrameType::kVideoFrameDelta}));
7872
7873 // Re-enable top layer.
7874 // Encoder should be re-initialized. Next frame should be key frame.
7875 config.simulcast_layers[number_layers - 1].active = true;
7876 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7877 sink_.SetNumExpectedLayers(number_layers);
7878 timestamp_ms += kFrameIntervalMs;
7879 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7880 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117881 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467882 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7883 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7884 VideoFrameType::kVideoFrameKey,
7885 VideoFrameType::kVideoFrameKey}));
7886
7887 // Top layer max rate change.
7888 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7889 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7890 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7891 sink_.SetNumExpectedLayers(number_layers);
7892 timestamp_ms += kFrameIntervalMs;
7893 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7894 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117895 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467896 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7897 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7898 VideoFrameType::kVideoFrameDelta,
7899 VideoFrameType::kVideoFrameDelta}));
7900
7901 // Top layer resolution change.
7902 // Encoder should be re-initialized. Next frame should be key frame.
7903 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7904 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7905 sink_.SetNumExpectedLayers(number_layers);
7906 timestamp_ms += kFrameIntervalMs;
7907 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7908 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 08:07:117909 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 07:22:467910 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7911 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7912 VideoFrameType::kVideoFrameKey,
7913 VideoFrameType::kVideoFrameKey}));
7914 video_stream_encoder_->Stop();
7915}
7916
Henrik Boström1124ed12021-02-25 09:30:397917TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7918 const int kFrameWidth = 1280;
7919 const int kFrameHeight = 720;
7920
7921 SetUp();
7922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117923 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 09:30:397924
7925 // Capturing a frame should reconfigure the encoder and expose the encoder
7926 // resolution, which is the same as the input frame.
7927 int64_t timestamp_ms = kFrameIntervalMs;
7928 video_source_.IncomingCapturedFrame(
7929 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7930 WaitForEncodedFrame(timestamp_ms);
7931 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7932 EXPECT_THAT(video_source_.sink_wants().resolutions,
7933 ::testing::ElementsAreArray(
7934 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7935
7936 video_stream_encoder_->Stop();
7937}
7938
7939TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7940 // Pick downscale factors such that we never encode at full resolution - this
7941 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 14:22:117942 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 09:30:397943 // encoder should not ask for the frame resolution. This allows video frames
7944 // to have the appearence of one resolution but optimize its internal buffers
7945 // for what is actually encoded.
7946 const size_t kNumSimulcastLayers = 3u;
7947 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7948 const int kFrameWidth = 1280;
7949 const int kFrameHeight = 720;
7950 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7951 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7952 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7953 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7954 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7955 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7956
7957 VideoEncoderConfig config;
7958 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7959 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7960 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7961 config.simulcast_layers[i].active = true;
7962 }
7963 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:437964 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 09:30:397965 "VP8", /*max qp*/ 56, /*screencast*/ false,
7966 /*screenshare enabled*/ false);
7967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:117968 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7969 0, 0, 0);
Henrik Boström1124ed12021-02-25 09:30:397970
7971 // Capture a frame with all layers active.
7972 int64_t timestamp_ms = kFrameIntervalMs;
7973 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7974 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7975 video_source_.IncomingCapturedFrame(
7976 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7977 WaitForEncodedFrame(timestamp_ms);
7978 // Expect encoded resolutions to match the expected simulcast layers.
7979 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7980 EXPECT_THAT(
7981 video_source_.sink_wants().resolutions,
7982 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7983
7984 // Capture a frame with one of the layers inactive.
7985 timestamp_ms += kFrameIntervalMs;
7986 config.simulcast_layers[2].active = false;
7987 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7988 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7989 video_source_.IncomingCapturedFrame(
7990 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7991 WaitForEncodedFrame(timestamp_ms);
7992
7993 // Expect encoded resolutions to match the expected simulcast layers.
7994 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7995 EXPECT_THAT(video_source_.sink_wants().resolutions,
7996 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7997
7998 // Capture a frame with all but one layer turned off.
7999 timestamp_ms += kFrameIntervalMs;
8000 config.simulcast_layers[1].active = false;
8001 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8002 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8003 video_source_.IncomingCapturedFrame(
8004 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8005 WaitForEncodedFrame(timestamp_ms);
8006
8007 // Expect encoded resolutions to match the expected simulcast layers.
8008 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8009 EXPECT_THAT(video_source_.sink_wants().resolutions,
8010 ::testing::ElementsAreArray({kLayer0Size}));
8011
8012 video_stream_encoder_->Stop();
8013}
8014
Sergey Silkin0e42cf72021-03-15 09:12:578015TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 09:12:578016 ResetEncoder("VP8", 1, 1, 1, false);
8017
Niels Möller8b692902021-06-14 10:04:578018 // Force encoder reconfig.
8019 video_source_.IncomingCapturedFrame(
8020 CreateFrame(1, codec_width_, codec_height_));
8021 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8022
Sergey Silkin0e42cf72021-03-15 09:12:578023 // Set QP on encoded frame and pass the frame to encode complete callback.
8024 // Since QP is present QP parsing won't be triggered and the original value
8025 // should be kept.
8026 EncodedImage encoded_image;
8027 encoded_image.qp_ = 123;
8028 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8029 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8030 CodecSpecificInfo codec_info;
8031 codec_info.codecType = kVideoCodecVP8;
8032 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8033 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8034 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8035 video_stream_encoder_->Stop();
8036}
8037
8038TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 09:12:578039 ResetEncoder("VP8", 1, 1, 1, false);
8040
Niels Möller8b692902021-06-14 10:04:578041 // Force encoder reconfig.
8042 video_source_.IncomingCapturedFrame(
8043 CreateFrame(1, codec_width_, codec_height_));
8044 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8045
Sergey Silkin0e42cf72021-03-15 09:12:578046 // Pass an encoded frame without QP to encode complete callback. QP should be
8047 // parsed and set.
8048 EncodedImage encoded_image;
8049 encoded_image.qp_ = -1;
8050 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8051 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8052 CodecSpecificInfo codec_info;
8053 codec_info.codecType = kVideoCodecVP8;
8054 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8055 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8056 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8057 video_stream_encoder_->Stop();
8058}
8059
8060TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
8061 webrtc::test::ScopedFieldTrials field_trials(
8062 "WebRTC-QpParsingKillSwitch/Enabled/");
8063
Sergey Silkin0e42cf72021-03-15 09:12:578064 ResetEncoder("VP8", 1, 1, 1, false);
8065
Niels Möller8b692902021-06-14 10:04:578066 // Force encoder reconfig.
8067 video_source_.IncomingCapturedFrame(
8068 CreateFrame(1, codec_width_, codec_height_));
8069 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8070
Sergey Silkin0e42cf72021-03-15 09:12:578071 EncodedImage encoded_image;
8072 encoded_image.qp_ = -1;
8073 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8074 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8075 CodecSpecificInfo codec_info;
8076 codec_info.codecType = kVideoCodecVP8;
8077 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8078 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8079 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8080 video_stream_encoder_->Stop();
8081}
8082
Sergey Silkind19e3b92021-03-16 10:05:308083TEST_F(VideoStreamEncoderTest,
8084 QualityScalingNotAllowed_QualityScalingDisabled) {
8085 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8086
8087 // Disable scaling settings in encoder info.
8088 fake_encoder_.SetQualityScaling(false);
8089 // Disable quality scaling in encoder config.
8090 video_encoder_config.is_quality_scaling_allowed = false;
8091 ConfigureEncoder(std::move(video_encoder_config));
8092
8093 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118094 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:308095
8096 test::FrameForwarder source;
8097 video_stream_encoder_->SetSource(
8098 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8099 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8100 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8101
8102 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8103 WaitForEncodedFrame(1);
8104 video_stream_encoder_->TriggerQualityLow();
8105 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8106
8107 video_stream_encoder_->Stop();
8108}
8109
Qiu Jianlinb54cfde2021-07-29 22:48:038110TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8111 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8112
8113 // Disable scaling settings in encoder info.
8114 fake_encoder_.SetQualityScaling(false);
8115 // Set QP trusted in encoder info.
8116 fake_encoder_.SetIsQpTrusted(true);
8117 // Enable quality scaling in encoder config.
8118 video_encoder_config.is_quality_scaling_allowed = false;
8119 ConfigureEncoder(std::move(video_encoder_config));
8120
8121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118122 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-29 22:48:038123
8124 test::FrameForwarder source;
8125 video_stream_encoder_->SetSource(
8126 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8127 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8128 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8129
8130 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8131 WaitForEncodedFrame(1);
8132 video_stream_encoder_->TriggerQualityLow();
8133 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8134
8135 video_stream_encoder_->Stop();
8136}
8137
Shuhai Pengf2707702021-09-29 09:19:448138TEST_F(VideoStreamEncoderTest,
8139 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8140 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8141
8142 // Disable scaling settings in encoder info.
8143 fake_encoder_.SetQualityScaling(false);
8144 // Set QP trusted in encoder info.
8145 fake_encoder_.SetIsQpTrusted(true);
8146 // Enable quality scaling in encoder config.
8147 video_encoder_config.is_quality_scaling_allowed = false;
8148 ConfigureEncoder(std::move(video_encoder_config));
8149
8150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118151 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448152
8153 test::FrameForwarder source;
8154 video_stream_encoder_->SetSource(
8155 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8156 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8157 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8158
8159 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8160 WaitForEncodedFrame(1);
8161 video_stream_encoder_->TriggerQualityLow();
8162 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8163
8164 video_stream_encoder_->Stop();
8165}
8166
8167TEST_F(VideoStreamEncoderTest,
8168 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8169 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8170
8171 // Disable scaling settings in encoder info.
8172 fake_encoder_.SetQualityScaling(false);
8173 // Set QP trusted in encoder info.
8174 fake_encoder_.SetIsQpTrusted(false);
8175 // Enable quality scaling in encoder config.
8176 video_encoder_config.is_quality_scaling_allowed = false;
8177 ConfigureEncoder(std::move(video_encoder_config));
8178
8179 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118180 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448181
8182 test::FrameForwarder source;
8183 video_stream_encoder_->SetSource(
8184 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8185 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8186 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8187
8188 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8189 WaitForEncodedFrame(1);
8190 video_stream_encoder_->TriggerQualityLow();
8191 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8192
8193 video_stream_encoder_->Stop();
8194}
8195
8196TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8197 // Set QP trusted in encoder info.
8198 fake_encoder_.SetIsQpTrusted(false);
8199
8200 const int MinEncBitrateKbps = 30;
8201 const int MaxEncBitrateKbps = 100;
8202 const int MinStartBitrateKbp = 50;
8203 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8204 /*frame_size_pixels=*/codec_width_ * codec_height_,
8205 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8206 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8207 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8208
8209 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118210 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448211
8212 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8213
8214 VideoEncoderConfig video_encoder_config;
8215 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8216 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8217 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8218 MinEncBitrateKbps * 1000;
8219 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8220 kMaxPayloadLength);
8221
8222 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8223 WaitForEncodedFrame(1);
8224 EXPECT_EQ(
8225 MaxEncBitrateKbps,
8226 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8227 EXPECT_EQ(
8228 MinEncBitrateKbps,
8229 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8230
8231 video_stream_encoder_->Stop();
8232}
8233
8234TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8235 // Set QP trusted in encoder info.
8236 fake_encoder_.SetIsQpTrusted(false);
8237
8238 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8239 EncoderInfoSettings::
8240 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8241 codec_width_ * codec_height_,
8242 EncoderInfoSettings::
8243 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8244 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8245
8246 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8247 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8248 const int TargetEncBitrate = MaxEncBitrate;
8249 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8250 DataRate::BitsPerSec(TargetEncBitrate),
8251 DataRate::BitsPerSec(TargetEncBitrate),
8252 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8253
8254 VideoEncoderConfig video_encoder_config;
8255 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8256 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8257 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8258 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8259 kMaxPayloadLength);
8260
8261 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8262 WaitForEncodedFrame(1);
8263 EXPECT_EQ(
8264 MaxEncBitrate / 1000,
8265 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8266 EXPECT_EQ(
8267 MinEncBitrate / 1000,
8268 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8269
8270 video_stream_encoder_->Stop();
8271}
8272
Sergey Silkind19e3b92021-03-16 10:05:308273#if !defined(WEBRTC_IOS)
8274// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8275// disabled by default on iOS.
8276TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8277 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8278
8279 // Disable scaling settings in encoder info.
8280 fake_encoder_.SetQualityScaling(false);
8281 // Enable quality scaling in encoder config.
8282 video_encoder_config.is_quality_scaling_allowed = true;
8283 ConfigureEncoder(std::move(video_encoder_config));
8284
8285 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118286 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:308287
8288 test::FrameForwarder source;
8289 video_stream_encoder_->SetSource(
8290 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8291 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8292 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8293
8294 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8295 WaitForEncodedFrame(1);
8296 video_stream_encoder_->TriggerQualityLow();
8297 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8298
8299 video_stream_encoder_->Stop();
8300}
Qiu Jianlinb54cfde2021-07-29 22:48:038301
8302TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8303 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8304
8305 // Disable scaling settings in encoder info.
8306 fake_encoder_.SetQualityScaling(false);
8307 // Set QP trusted in encoder info.
8308 fake_encoder_.SetIsQpTrusted(true);
8309 // Enable quality scaling in encoder config.
8310 video_encoder_config.is_quality_scaling_allowed = true;
8311 ConfigureEncoder(std::move(video_encoder_config));
8312
8313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118314 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-29 22:48:038315
8316 test::FrameForwarder source;
8317 video_stream_encoder_->SetSource(
8318 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8319 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8320 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8321
8322 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8323 WaitForEncodedFrame(1);
8324 video_stream_encoder_->TriggerQualityLow();
8325 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8326
8327 video_stream_encoder_->Stop();
8328}
Shuhai Pengf2707702021-09-29 09:19:448329
8330TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8331 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8332
8333 // Disable scaling settings in encoder info.
8334 fake_encoder_.SetQualityScaling(false);
8335 // Set QP not trusted in encoder info.
8336 fake_encoder_.SetIsQpTrusted(false);
8337 // Enable quality scaling in encoder config.
8338 video_encoder_config.is_quality_scaling_allowed = true;
8339 ConfigureEncoder(std::move(video_encoder_config));
8340
8341 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118342 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448343
8344 test::FrameForwarder source;
8345 video_stream_encoder_->SetSource(
8346 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8347 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8348 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8349
8350 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8351 WaitForEncodedFrame(1);
8352 video_stream_encoder_->TriggerQualityLow();
8353 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8354 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8355 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8356
8357 video_stream_encoder_->Stop();
8358}
8359
8360TEST_F(VideoStreamEncoderTest,
8361 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8362 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8363
8364 // Disable scaling settings in encoder info.
8365 fake_encoder_.SetQualityScaling(false);
8366 // Set QP trusted in encoder info.
8367 fake_encoder_.SetIsQpTrusted(true);
8368 // Enable quality scaling in encoder config.
8369 video_encoder_config.is_quality_scaling_allowed = true;
8370 ConfigureEncoder(std::move(video_encoder_config));
8371
8372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118373 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448374
8375 test::FrameForwarder source;
8376 video_stream_encoder_->SetSource(
8377 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8378 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8379 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8380
8381 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8382 WaitForEncodedFrame(1);
8383 video_stream_encoder_->TriggerQualityLow();
8384 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8385 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8386
8387 video_stream_encoder_->Stop();
8388}
8389
8390TEST_F(VideoStreamEncoderTest,
8391 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8392 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8393
8394 // Disable scaling settings in encoder info.
8395 fake_encoder_.SetQualityScaling(false);
8396 // Set QP trusted in encoder info.
8397 fake_encoder_.SetIsQpTrusted(false);
8398 // Enable quality scaling in encoder config.
8399 video_encoder_config.is_quality_scaling_allowed = true;
8400 ConfigureEncoder(std::move(video_encoder_config));
8401
8402 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118403 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 09:19:448404
8405 test::FrameForwarder source;
8406 video_stream_encoder_->SetSource(
8407 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8408 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8409 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8410
8411 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8412 WaitForEncodedFrame(1);
8413 video_stream_encoder_->TriggerQualityLow();
8414 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8415
8416 video_stream_encoder_->Stop();
8417}
8418
Sergey Silkind19e3b92021-03-16 10:05:308419#endif
8420
Henrik Boström56db9ff2021-03-24 08:06:458421// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8422class VideoStreamEncoderWithRealEncoderTest
8423 : public VideoStreamEncoderTest,
8424 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8425 public:
8426 VideoStreamEncoderWithRealEncoderTest()
8427 : VideoStreamEncoderTest(),
8428 codec_type_(std::get<0>(GetParam())),
8429 allow_i420_conversion_(std::get<1>(GetParam())) {}
8430
8431 void SetUp() override {
8432 VideoStreamEncoderTest::SetUp();
8433 std::unique_ptr<VideoEncoder> encoder;
8434 switch (codec_type_) {
8435 case kVideoCodecVP8:
8436 encoder = VP8Encoder::Create();
8437 break;
8438 case kVideoCodecVP9:
8439 encoder = VP9Encoder::Create();
8440 break;
8441 case kVideoCodecAV1:
philipel95701502022-01-18 17:47:528442 encoder = CreateLibaomAv1EncoderIfSupported();
Henrik Boström56db9ff2021-03-24 08:06:458443 break;
8444 case kVideoCodecH264:
8445 encoder =
8446 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8447 break;
8448 case kVideoCodecMultiplex:
8449 mock_encoder_factory_for_multiplex_ =
8450 std::make_unique<MockVideoEncoderFactory>();
8451 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8452 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8453 .WillRepeatedly([] { return VP8Encoder::Create(); });
8454 encoder = std::make_unique<MultiplexEncoderAdapter>(
8455 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8456 false);
8457 break;
8458 default:
Artem Titovd3251962021-11-15 15:57:078459 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 08:06:458460 }
8461 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8462 }
8463
8464 void TearDown() override {
8465 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 14:22:118466 // Ensure `video_stream_encoder_` is destroyed before
8467 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 08:06:458468 video_stream_encoder_.reset();
8469 VideoStreamEncoderTest::TearDown();
8470 }
8471
8472 protected:
8473 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8474 std::unique_ptr<VideoEncoder> encoder) {
8475 // Configure VSE to use the encoder.
8476 encoder_ = std::move(encoder);
8477 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8478 encoder_.get(), &encoder_selector_);
8479 video_send_config_.encoder_settings.encoder_factory =
8480 encoder_proxy_factory_.get();
8481 VideoEncoderConfig video_encoder_config;
8482 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8483 video_encoder_config_ = video_encoder_config.Copy();
8484 ConfigureEncoder(video_encoder_config_.Copy());
8485
8486 // Set bitrate to ensure frame is not dropped.
8487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118488 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 08:06:458489 }
8490
8491 const VideoCodecType codec_type_;
8492 const bool allow_i420_conversion_;
8493 NiceMock<MockEncoderSelector> encoder_selector_;
8494 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8495 std::unique_ptr<VideoEncoder> encoder_;
8496 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8497};
8498
8499TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8500 auto native_i420_frame = test::CreateMappableNativeFrame(
8501 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8502 video_source_.IncomingCapturedFrame(native_i420_frame);
8503 WaitForEncodedFrame(codec_width_, codec_height_);
8504
8505 auto mappable_native_buffer =
8506 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8507 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8508 mappable_native_buffer->GetMappedFramedBuffers();
8509 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8510 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8511 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8512 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8513}
8514
8515TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8516 auto native_nv12_frame = test::CreateMappableNativeFrame(
8517 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8518 video_source_.IncomingCapturedFrame(native_nv12_frame);
8519 WaitForEncodedFrame(codec_width_, codec_height_);
8520
8521 auto mappable_native_buffer =
8522 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8523 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8524 mappable_native_buffer->GetMappedFramedBuffers();
8525 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8526 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8527 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8528 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8529
8530 if (!allow_i420_conversion_) {
8531 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8532 }
8533}
8534
Erik Språng7444b192021-06-02 12:02:138535TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8536 if (codec_type_ == kVideoCodecMultiplex) {
8537 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8538 return;
8539 }
8540
8541 const size_t kNumSpatialLayers = 3u;
8542 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8543 const int kFrameWidth = 1280;
8544 const int kFrameHeight = 720;
8545 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8546 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8547 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8548 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8549 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8550 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8551
8552 VideoEncoderConfig config;
8553 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8554 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 08:07:118555 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 12:02:138556 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8557 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8558 vp9_settings.numberOfTemporalLayers = 3;
8559 vp9_settings.automaticResizeOn = false;
8560 config.encoder_specific_settings =
8561 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8562 vp9_settings);
8563 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8564 /*fps=*/30.0,
8565 /*first_active_layer=*/0,
8566 /*num_spatial_layers=*/3,
8567 /*num_temporal_layers=*/3,
8568 /*is_screenshare=*/false);
8569 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8570 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 08:07:118571 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 12:02:138572 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8573 /*fps=*/30.0,
8574 /*first_active_layer=*/0,
8575 /*num_spatial_layers=*/3,
8576 /*num_temporal_layers=*/3,
8577 /*is_screenshare=*/false);
8578 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8579 } else {
8580 // Simulcast for VP8/H264.
8581 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8582 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8583 config.simulcast_layers[i].scale_resolution_down_by =
8584 kDownscaleFactors[i];
8585 config.simulcast_layers[i].active = true;
8586 }
8587 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8588 // Turn off frame dropping to prevent flakiness.
8589 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8590 h264_settings.frameDroppingOn = false;
8591 config.encoder_specific_settings = rtc::make_ref_counted<
8592 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8593 }
8594 }
8595
8596 auto set_layer_active = [&](int layer_idx, bool active) {
8597 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8598 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8599 config.spatial_layers[layer_idx].active = active;
8600 } else {
8601 config.simulcast_layers[layer_idx].active = active;
8602 }
8603 };
8604
8605 config.video_stream_factory =
8606 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8607 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8608 /*screencast*/ false,
8609 /*screenshare enabled*/ false);
8610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 08:07:118611 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8612 0, 0, 0);
Erik Språng7444b192021-06-02 12:02:138613
8614 // Capture a frame with all layers active.
8615 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8616 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8617 int64_t timestamp_ms = kFrameIntervalMs;
8618 video_source_.IncomingCapturedFrame(
8619 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8620
8621 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8622 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8623
8624 // Capture a frame with one of the layers inactive.
8625 set_layer_active(2, false);
8626 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8627 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8628 timestamp_ms += kFrameIntervalMs;
8629 video_source_.IncomingCapturedFrame(
8630 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8631 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8632
8633 // New target bitrates signaled based on lower resolution.
8634 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8636 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8637 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8638
8639 // Re-enable the top layer.
8640 set_layer_active(2, true);
8641 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8642 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8643 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8644
8645 // Bitrate target adjusted back up to enable HD layer...
8646 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8647 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8648 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8649 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8650
8651 // ...then add a new frame.
8652 timestamp_ms += kFrameIntervalMs;
8653 video_source_.IncomingCapturedFrame(
8654 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8655 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8656 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8657
8658 video_stream_encoder_->Stop();
8659}
8660
Henrik Boström56db9ff2021-03-24 08:06:458661std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8662 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8663 VideoCodecType codec_type = std::get<0>(info.param);
8664 bool allow_i420_conversion = std::get<1>(info.param);
8665 std::string str;
8666 switch (codec_type) {
8667 case kVideoCodecGeneric:
8668 str = "Generic";
8669 break;
8670 case kVideoCodecVP8:
8671 str = "VP8";
8672 break;
8673 case kVideoCodecVP9:
8674 str = "VP9";
8675 break;
8676 case kVideoCodecAV1:
8677 str = "AV1";
8678 break;
8679 case kVideoCodecH264:
8680 str = "H264";
8681 break;
8682 case kVideoCodecMultiplex:
8683 str = "Multiplex";
8684 break;
8685 default:
Artem Titovd3251962021-11-15 15:57:078686 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 08:06:458687 }
8688 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8689 return str;
8690}
8691
8692constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8693 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8694constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8695 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8696constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8697 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8698constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8699 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8700#if defined(WEBRTC_USE_H264)
8701constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8702 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8703
8704// The windows compiler does not tolerate #if statements inside the
8705// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8706// and without H264).
8707INSTANTIATE_TEST_SUITE_P(
8708 All,
8709 VideoStreamEncoderWithRealEncoderTest,
8710 ::testing::Values(kVP8DisallowConversion,
8711 kVP9DisallowConversion,
8712 kAV1AllowConversion,
8713 kMultiplexDisallowConversion,
8714 kH264AllowConversion),
8715 TestParametersVideoCodecAndAllowI420ConversionToString);
8716#else
8717INSTANTIATE_TEST_SUITE_P(
8718 All,
8719 VideoStreamEncoderWithRealEncoderTest,
8720 ::testing::Values(kVP8DisallowConversion,
8721 kVP9DisallowConversion,
8722 kAV1AllowConversion,
8723 kMultiplexDisallowConversion),
8724 TestParametersVideoCodecAndAllowI420ConversionToString);
8725#endif
8726
Åsa Persson4d4f62f2021-09-12 14:14:488727class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8728 protected:
8729 void RunTest(const std::vector<VideoStream>& configs,
8730 const int expected_num_init_encode) {
8731 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 08:07:118732 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 14:14:488733 InsertFrameAndWaitForEncoded();
8734 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8735 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 08:07:118736 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8737 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 14:14:488738
8739 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8740 ConfigureEncoder(configs[1]);
8741 InsertFrameAndWaitForEncoded();
8742 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8743 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 08:07:118744 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 14:14:488745 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 08:07:118746 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 14:14:488747
8748 video_stream_encoder_->Stop();
8749 }
8750
8751 void ConfigureEncoder(const VideoStream& stream) {
8752 VideoEncoderConfig config;
8753 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8754 config.max_bitrate_bps = stream.max_bitrate_bps;
8755 config.simulcast_layers[0] = stream;
8756 config.video_stream_factory =
8757 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8758 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8759 /*conference_mode=*/false);
8760 video_stream_encoder_->ConfigureEncoder(std::move(config),
8761 kMaxPayloadLength);
8762 }
8763
8764 void OnBitrateUpdated(DataRate bitrate) {
8765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8766 bitrate, bitrate, bitrate, 0, 0, 0);
8767 }
8768
8769 void InsertFrameAndWaitForEncoded() {
8770 timestamp_ms_ += kFrameIntervalMs;
8771 video_source_.IncomingCapturedFrame(
8772 CreateFrame(timestamp_ms_, kWidth, kHeight));
8773 sink_.WaitForEncodedFrame(timestamp_ms_);
8774 }
8775
8776 void ExpectEqual(const VideoCodec& actual,
8777 const VideoStream& expected) const {
8778 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8779 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8780 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8781 static_cast<unsigned int>(expected.min_bitrate_bps));
8782 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8783 static_cast<unsigned int>(expected.max_bitrate_bps));
8784 EXPECT_EQ(actual.simulcastStream[0].width,
8785 kWidth / expected.scale_resolution_down_by);
8786 EXPECT_EQ(actual.simulcastStream[0].height,
8787 kHeight / expected.scale_resolution_down_by);
8788 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8789 expected.num_temporal_layers);
8790 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8791 }
8792
8793 VideoStream DefaultConfig() const {
8794 VideoStream stream;
8795 stream.max_framerate = 25;
8796 stream.min_bitrate_bps = 35000;
8797 stream.max_bitrate_bps = 900000;
8798 stream.scale_resolution_down_by = 1.0;
8799 stream.num_temporal_layers = 1;
8800 stream.bitrate_priority = 1.0;
8801 stream.scalability_mode = "";
8802 return stream;
8803 }
8804
8805 const int kWidth = 640;
8806 const int kHeight = 360;
8807 int64_t timestamp_ms_ = 0;
8808};
8809
8810TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8811 VideoStream config1 = DefaultConfig();
8812 VideoStream config2 = config1;
8813 config2.max_framerate++;
8814
8815 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8816}
8817
8818TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8819 VideoStream config1 = DefaultConfig();
8820 VideoStream config2 = config1;
8821 config2.min_bitrate_bps += 10000;
8822
8823 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8824}
8825
8826TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8827 VideoStream config1 = DefaultConfig();
8828 VideoStream config2 = config1;
8829 config2.max_bitrate_bps += 100000;
8830
8831 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8832}
8833
8834TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8835 VideoStream config1 = DefaultConfig();
8836 VideoStream config2 = config1;
8837 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8838
8839 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8840}
8841
8842TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8843 VideoStream config1 = DefaultConfig();
8844 VideoStream config2 = config1;
8845 config2.scale_resolution_down_by *= 2;
8846
8847 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8848}
8849
8850TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8851 VideoStream config1 = DefaultConfig();
8852 VideoStream config2 = config1;
8853 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8854
8855 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8856}
8857
8858TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8859 VideoStream config1 = DefaultConfig();
8860 VideoStream config2 = config1;
8861 config2.scalability_mode = "L1T2";
8862
8863 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8864}
8865
Tommi62b01db2022-01-25 22:41:228866// Simple test that just creates and then immediately destroys an encoder.
8867// The purpose of the test is to make sure that nothing bad happens if the
8868// initialization step on the encoder queue, doesn't run.
8869TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
8870 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
8871 public:
8872 SuperLazyTaskQueue() = default;
8873 ~SuperLazyTaskQueue() override = default;
8874
8875 private:
8876 void Delete() override { delete this; }
8877 void PostTask(std::unique_ptr<QueuedTask> task) override {
8878 // meh.
8879 }
8880 void PostDelayedTask(std::unique_ptr<QueuedTask> task,
8881 uint32_t milliseconds) override {
8882 ASSERT_TRUE(false);
8883 }
8884 };
8885
8886 // Lots of boiler plate.
8887 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
8888 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
8889 time_controller.GetClock(), VideoSendStream::Config(nullptr),
8890 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
8891 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
8892 time_controller.GetClock());
8893 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
8894 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
8895 CreateBuiltinVideoBitrateAllocatorFactory();
8896 VideoStreamEncoderSettings encoder_settings{
8897 VideoEncoder::Capabilities(/*loss_notification=*/false)};
8898 encoder_settings.encoder_factory = &encoder_factory;
8899 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
8900
8901 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8902 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
8903
8904 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
8905 encoder_queue(new SuperLazyTaskQueue());
8906
8907 // Construct a VideoStreamEncoder instance and let it go out of scope without
8908 // doing anything else (including calling Stop()). This should be fine since
8909 // the posted init task will simply be deleted.
8910 auto encoder = std::make_unique<VideoStreamEncoder>(
8911 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
8912 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get()),
8913 std::move(adapter), std::move(encoder_queue),
8914 VideoStreamEncoder::BitrateAllocationCallbackType::
8915 kVideoBitrateAllocation);
8916}
8917
Markus Handellb4e96d42021-11-05 11:00:558918TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8919 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8920 auto* adapter_ptr = adapter.get();
8921 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 10:37:168922 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8923 nullptr;
8924 EXPECT_CALL(*adapter_ptr, Initialize)
8925 .WillOnce(Invoke([&video_stream_encoder_callback](
8926 FrameCadenceAdapterInterface::Callback* callback) {
8927 video_stream_encoder_callback = callback;
8928 }));
8929 TaskQueueBase* encoder_queue = nullptr;
8930 auto video_stream_encoder =
8931 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 11:00:558932
Markus Handelle59fee82021-12-23 08:29:238933 // First a call before we know the frame size and hence cannot compute the
8934 // number of simulcast layers.
8935 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8936 &FrameCadenceAdapterInterface::
8937 ZeroHertzModeParams::num_simulcast_layers,
8938 Eq(0)))));
Markus Handellb4e96d42021-11-05 11:00:558939 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 10:37:168940 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 11:00:558941 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8942 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 08:29:238943 factory.DepleteTaskQueues();
8944
8945 // Then a call as we've computed the number of simulcast layers after a passed
8946 // frame.
8947 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8948 &FrameCadenceAdapterInterface::
8949 ZeroHertzModeParams::num_simulcast_layers,
8950 Gt(0)))));
Markus Handell8d87c462021-12-16 10:37:168951 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 15:07:018952 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 11:00:558953 Mock::VerifyAndClearExpectations(adapter_ptr);
8954
Markus Handelle59fee82021-12-23 08:29:238955 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 10:37:168956 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 11:00:558957 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 10:37:168958 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 11:00:558959 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8960 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 10:37:168961 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 15:07:018962 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 11:00:558963}
8964
8965TEST(VideoStreamEncoderFrameCadenceTest,
8966 ForwardsFramesIntoFrameCadenceAdapter) {
8967 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8968 auto* adapter_ptr = adapter.get();
8969 test::FrameForwarder video_source;
8970 SimpleVideoStreamEncoderFactory factory;
8971 auto video_stream_encoder = factory.Create(std::move(adapter));
8972 video_stream_encoder->SetSource(
8973 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8974
8975 EXPECT_CALL(*adapter_ptr, OnFrame);
8976 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8977 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 10:37:168978 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 11:00:558979}
8980
Markus Handellee225432021-11-29 11:35:128981TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
8982 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8983 auto* adapter_ptr = adapter.get();
8984 test::FrameForwarder video_source;
8985 SimpleVideoStreamEncoderFactory factory;
8986 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8987 nullptr;
8988 EXPECT_CALL(*adapter_ptr, Initialize)
8989 .WillOnce(Invoke([&video_stream_encoder_callback](
8990 FrameCadenceAdapterInterface::Callback* callback) {
8991 video_stream_encoder_callback = callback;
8992 }));
8993 TaskQueueBase* encoder_queue = nullptr;
8994 auto video_stream_encoder =
8995 factory.Create(std::move(adapter), &encoder_queue);
8996
8997 // This is just to make the VSE operational. We'll feed a frame directly by
8998 // the callback interface.
8999 video_stream_encoder->SetSource(
9000 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9001
9002 VideoEncoderConfig video_encoder_config;
9003 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9004 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9005 /*max_data_payload_length=*/1000);
9006
9007 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9008 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 10:37:169009 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 11:35:129010 factory.DepleteTaskQueues();
9011}
9012
Markus Handell8d87c462021-12-16 10:37:169013TEST(VideoStreamEncoderFrameCadenceTest,
9014 DeactivatesActivatesLayersOnBitrateChanges) {
9015 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9016 auto* adapter_ptr = adapter.get();
9017 SimpleVideoStreamEncoderFactory factory;
9018 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9019 nullptr;
9020 EXPECT_CALL(*adapter_ptr, Initialize)
9021 .WillOnce(Invoke([&video_stream_encoder_callback](
9022 FrameCadenceAdapterInterface::Callback* callback) {
9023 video_stream_encoder_callback = callback;
9024 }));
9025 TaskQueueBase* encoder_queue = nullptr;
9026 auto video_stream_encoder =
9027 factory.Create(std::move(adapter), &encoder_queue);
9028
9029 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9030 // {150000, 450000}.
9031 VideoEncoderConfig video_encoder_config;
9032 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9033 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9034 kMaxPayloadLength);
9035 // Ensure an encoder is created.
9036 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9037
9038 // Both layers enabled at 1 MBit/s.
9039 video_stream_encoder->OnBitrateUpdated(
9040 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9041 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9042 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9043 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9044 factory.DepleteTaskQueues();
9045 Mock::VerifyAndClearExpectations(adapter_ptr);
9046
9047 // Layer 1 disabled at 200 KBit/s.
9048 video_stream_encoder->OnBitrateUpdated(
9049 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9050 DataRate::KilobitsPerSec(200), 0, 0, 0);
9051 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9052 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9053 factory.DepleteTaskQueues();
9054 Mock::VerifyAndClearExpectations(adapter_ptr);
9055
9056 // All layers off at suspended video.
9057 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9058 DataRate::Zero(), 0, 0, 0);
9059 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9060 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9061 factory.DepleteTaskQueues();
9062 Mock::VerifyAndClearExpectations(adapter_ptr);
9063
9064 // Both layers enabled again back at 1 MBit/s.
9065 video_stream_encoder->OnBitrateUpdated(
9066 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9067 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9068 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9069 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9070 factory.DepleteTaskQueues();
9071}
9072
9073TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9074 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9075 auto* adapter_ptr = adapter.get();
9076 SimpleVideoStreamEncoderFactory factory;
9077 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9078 nullptr;
9079 EXPECT_CALL(*adapter_ptr, Initialize)
9080 .WillOnce(Invoke([&video_stream_encoder_callback](
9081 FrameCadenceAdapterInterface::Callback* callback) {
9082 video_stream_encoder_callback = callback;
9083 }));
9084 TaskQueueBase* encoder_queue = nullptr;
9085 auto video_stream_encoder =
9086 factory.Create(std::move(adapter), &encoder_queue);
9087
9088 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9089 VideoEncoderConfig video_encoder_config;
9090 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9091 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9092 kMaxPayloadLength);
9093 video_stream_encoder->OnBitrateUpdated(
9094 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9095 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9096
9097 // Pass a frame which has unconverged results.
9098 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9099 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9100 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9101 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9102 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9103 CodecSpecificInfo codec_specific;
9104 codec_specific.codecType = kVideoCodecGeneric;
9105 return codec_specific;
9106 }));
9107 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9108 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9109 factory.DepleteTaskQueues();
9110 Mock::VerifyAndClearExpectations(adapter_ptr);
9111 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9112
9113 // Pass a frame which converges in layer 0 and not in layer 1.
9114 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9115 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9116 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9117 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9118 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9119 CodecSpecificInfo codec_specific;
9120 codec_specific.codecType = kVideoCodecGeneric;
9121 return codec_specific;
9122 }));
9123 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9124 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9125 factory.DepleteTaskQueues();
9126 Mock::VerifyAndClearExpectations(adapter_ptr);
9127 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9128}
9129
Markus Handell2e0f4f02021-12-21 18:14:589130TEST(VideoStreamEncoderFrameCadenceTest,
9131 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9132 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9133 auto* adapter_ptr = adapter.get();
9134 MockVideoSourceInterface mock_source;
9135 SimpleVideoStreamEncoderFactory factory;
9136 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9137 nullptr;
9138 EXPECT_CALL(*adapter_ptr, Initialize)
9139 .WillOnce(Invoke([&video_stream_encoder_callback](
9140 FrameCadenceAdapterInterface::Callback* callback) {
9141 video_stream_encoder_callback = callback;
9142 }));
9143 TaskQueueBase* encoder_queue = nullptr;
9144 auto video_stream_encoder =
9145 factory.Create(std::move(adapter), &encoder_queue);
9146 video_stream_encoder->SetSource(
9147 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9148 VideoEncoderConfig config;
9149 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9150 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9151 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9152 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9153 // Ensure the encoder is set up.
9154 factory.DepleteTaskQueues();
9155
Markus Handell818e7fb2021-12-30 12:01:339156 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9157 .WillOnce(Invoke([video_stream_encoder_callback] {
9158 video_stream_encoder_callback->RequestRefreshFrame();
9159 }));
Markus Handell2e0f4f02021-12-21 18:14:589160 EXPECT_CALL(mock_source, RequestRefreshFrame);
9161 video_stream_encoder->SendKeyFrame();
9162 factory.DepleteTaskQueues();
9163 Mock::VerifyAndClearExpectations(adapter_ptr);
9164 Mock::VerifyAndClearExpectations(&mock_source);
9165
Markus Handell818e7fb2021-12-30 12:01:339166 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 18:14:589167 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9168 video_stream_encoder->SendKeyFrame();
9169 factory.DepleteTaskQueues();
9170}
9171
Markus Handell818e7fb2021-12-30 12:01:339172TEST(VideoStreamEncoderFrameCadenceTest,
9173 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9174 SimpleVideoStreamEncoderFactory factory;
9175 auto encoder_queue =
9176 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9177 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9178
9179 // Enables zero-hertz mode.
9180 test::ScopedFieldTrials field_trials("WebRTC-ZeroHertzScreenshare/Enabled/");
9181 auto adapter = FrameCadenceAdapterInterface::Create(
9182 factory.GetTimeController()->GetClock(), encoder_queue.get());
9183 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9184
9185 MockVideoSourceInterface mock_source;
9186 auto video_stream_encoder = factory.CreateWithEncoderQueue(
9187 std::move(adapter), std::move(encoder_queue));
9188
9189 video_stream_encoder->SetSource(
9190 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9191 VideoEncoderConfig config;
9192 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9193 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9194 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9195
9196 // Eventually expect a refresh frame request when requesting a key frame
9197 // before initializing zero-hertz mode. This can happen in reality because the
9198 // threads invoking key frame requests and constraints setup aren't
9199 // synchronized.
9200 EXPECT_CALL(mock_source, RequestRefreshFrame);
9201 video_stream_encoder->SendKeyFrame();
9202 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, 30});
9203 factory.DepleteTaskQueues();
9204}
9205
perkj26091b12016-09-01 08:17:409206} // namespace webrtc