blob: cbfd93e9e23f71b5bbf2082d338a9435807c632f [file] [log] [blame]
perkj26091b12016-09-01 08:17:401/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 08:30:3111#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 15:24:5913#include <algorithm>
perkj803d97f2016-11-01 18:45:4614#include <limits>
Danil Chapovalovd3ba2362019-04-10 15:01:2315#include <memory>
Henrik Boström56db9ff2021-03-24 08:06:4516#include <tuple>
Per512ecb32016-09-23 13:52:0617#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 14:30:1619#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 15:01:2320#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 09:20:0921#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 10:30:0022#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 08:06:4523#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 18:02:5624#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3125#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 07:15:1326#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 09:36:5527#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 16:10:5728#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 08:06:4529#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 12:57:5730#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 08:30:3131#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 10:56:2032#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 11:02:3633#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 15:26:0134#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 14:30:1635#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 13:59:1236#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 17:11:0037#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 15:50:0038#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 08:06:4539#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
40#include "modules/video_coding/codecs/h264/include/h264.h"
41#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
42#include "modules/video_coding/codecs/vp8/include/vp8.h"
43#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 17:32:3744#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 12:02:1345#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 10:24:3346#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 11:06:5947#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2348#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 09:39:5149#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 14:29:0950#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3151#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 17:11:0052#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 11:13:3253#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 15:20:1754#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 06:51:1055#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3156#include "test/encoder_settings.h"
57#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 08:51:4058#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 14:59:0059#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3160#include "test/gmock.h"
61#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 08:06:4562#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2363#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 07:07:2464#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3165#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 08:17:4066
67namespace webrtc {
68
sprang57c2fff2017-01-16 14:24:0269using ::testing::_;
philipeld9cc8c02019-09-16 12:53:4070using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 09:28:4171using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 12:08:3972using ::testing::Eq;
philipeld9cc8c02019-09-16 12:53:4073using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 12:08:3974using ::testing::Ge;
75using ::testing::Gt;
76using ::testing::Le;
77using ::testing::Lt;
philipel9b058032020-02-10 10:30:0078using ::testing::Matcher;
79using ::testing::NiceMock;
80using ::testing::Return;
Per Kjellander4190ce92020-12-15 16:24:5581using ::testing::SizeIs;
philipeld9cc8c02019-09-16 12:53:4082using ::testing::StrictMock;
kthelgason876222f2016-11-29 09:44:1183
perkj803d97f2016-11-01 18:45:4684namespace {
Åsa Persson8c1bf952018-09-13 08:42:1985const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 14:56:0086const int kQpLow = 1;
87const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 08:42:1988const int kMinFramerateFps = 2;
89const int kMinBalancedFramerateFps = 7;
90const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 12:37:0091const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 20:19:5392const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 12:18:3493const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 20:19:5394const uint32_t kSimulcastTargetBitrateBps = 3150000;
95const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 15:02:2296const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 11:21:0797const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 08:42:1998const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 08:48:4899const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 11:12:19100const VideoEncoder::ResolutionBitrateLimits
101 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
102const VideoEncoder::ResolutionBitrateLimits
103 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 12:37:00104
Mirta Dvornicic28f0eb22019-05-28 14:30:16105uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
106 0x00, 0x00, 0x03, 0x03, 0xF4,
107 0x05, 0x03, 0xC7, 0xE0, 0x1B,
108 0x41, 0x10, 0x8D, 0x00};
109
Sergey Silkin0e42cf72021-03-15 09:12:57110const uint8_t kCodedFrameVp8Qp25[] = {
111 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
112 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
113 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
114
perkj803d97f2016-11-01 18:45:46115class TestBuffer : public webrtc::I420Buffer {
116 public:
117 TestBuffer(rtc::Event* event, int width, int height)
118 : I420Buffer(width, height), event_(event) {}
119
120 private:
121 friend class rtc::RefCountedObject<TestBuffer>;
122 ~TestBuffer() override {
123 if (event_)
124 event_->Set();
125 }
126 rtc::Event* const event_;
127};
128
Henrik Boström56db9ff2021-03-24 08:06:45129// A fake native buffer that can't be converted to I420. Upon scaling, it
130// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 13:59:12131class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
132 public:
133 FakeNativeBuffer(rtc::Event* event, int width, int height)
134 : event_(event), width_(width), height_(height) {}
135 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
136 int width() const override { return width_; }
137 int height() const override { return height_; }
138 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
139 return nullptr;
140 }
Henrik Boström56db9ff2021-03-24 08:06:45141 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
142 int offset_x,
143 int offset_y,
144 int crop_width,
145 int crop_height,
146 int scaled_width,
147 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 17:21:43148 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
149 scaled_height);
Henrik Boström56db9ff2021-03-24 08:06:45150 }
Noah Richards51db4212019-06-12 13:59:12151
152 private:
153 friend class rtc::RefCountedObject<FakeNativeBuffer>;
154 ~FakeNativeBuffer() override {
155 if (event_)
156 event_->Set();
157 }
158 rtc::Event* const event_;
159 const int width_;
160 const int height_;
161};
162
Evan Shrubsole895556e2020-10-05 07:15:13163// A fake native buffer that is backed by an NV12 buffer.
164class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
165 public:
166 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
167 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
168
169 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
170 int width() const override { return nv12_buffer_->width(); }
171 int height() const override { return nv12_buffer_->height(); }
172 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
173 return nv12_buffer_->ToI420();
174 }
Evan Shrubsoleb556b082020-10-08 12:56:45175 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
176 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
177 if (absl::c_find(types, Type::kNV12) != types.end()) {
178 return nv12_buffer_;
179 }
180 return nullptr;
181 }
Evan Shrubsole895556e2020-10-05 07:15:13182 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
183
184 private:
185 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
186 ~FakeNV12NativeBuffer() override {
187 if (event_)
188 event_->Set();
189 }
190 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
191 rtc::Event* const event_;
192};
193
Niels Möller7dc26b72017-12-06 09:27:48194class CpuOveruseDetectorProxy : public OveruseFrameDetector {
195 public:
Niels Möllerd1f7eb62018-03-28 14:40:58196 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
197 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 16:49:07198 last_target_framerate_fps_(-1),
199 framerate_updated_event_(true /* manual_reset */,
200 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 09:27:48201 virtual ~CpuOveruseDetectorProxy() {}
202
203 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 11:13:32204 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48205 last_target_framerate_fps_ = framerate_fps;
206 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 16:49:07207 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 09:27:48208 }
209
210 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 11:13:32211 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48212 return last_target_framerate_fps_;
213 }
214
Niels Möller4db138e2018-04-19 07:04:13215 CpuOveruseOptions GetOptions() { return options_; }
216
Henrik Boström381d1092020-05-12 16:49:07217 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
218
Niels Möller7dc26b72017-12-06 09:27:48219 private:
Markus Handella3765182020-07-08 11:13:32220 Mutex lock_;
Niels Möller7dc26b72017-12-06 09:27:48221 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 16:49:07222 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 09:27:48223};
224
Henrik Boström0f0aa9c2020-06-02 11:02:36225class FakeVideoSourceRestrictionsListener
226 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 16:49:07227 public:
Henrik Boström0f0aa9c2020-06-02 11:02:36228 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 16:49:07229 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 11:02:36230 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 16:49:07231 RTC_DCHECK(was_restrictions_updated_);
232 }
233
234 rtc::Event* restrictions_updated_event() {
235 return &restrictions_updated_event_;
236 }
237
Henrik Boström0f0aa9c2020-06-02 11:02:36238 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 16:49:07239 void OnVideoSourceRestrictionsUpdated(
240 VideoSourceRestrictions restrictions,
241 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 09:47:46242 rtc::scoped_refptr<Resource> reason,
243 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 16:49:07244 was_restrictions_updated_ = true;
245 restrictions_updated_event_.Set();
246 }
247
248 private:
249 bool was_restrictions_updated_;
250 rtc::Event restrictions_updated_event_;
251};
252
Evan Shrubsole5fd40602020-05-25 14:19:54253auto WantsFps(Matcher<int> fps_matcher) {
254 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
255 fps_matcher);
256}
257
258auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
259 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
260 AllOf(max_pixel_matcher, Gt(0)));
261}
262
Evan Shrubsole5cd7eb82020-05-25 12:08:39263auto ResolutionMax() {
264 return AllOf(
Evan Shrubsole5fd40602020-05-25 14:19:54265 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 12:08:39266 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
267 Eq(absl::nullopt)));
268}
269
270auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 14:19:54271 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 12:08:39272}
273
274auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 14:19:54275 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 12:08:39276}
277
278auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54279 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 12:08:39280}
281
282auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54283 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 12:08:39284}
285
286auto FpsMaxResolutionMax() {
287 return AllOf(FpsMax(), ResolutionMax());
288}
289
290auto UnlimitedSinkWants() {
291 return AllOf(FpsUnlimited(), ResolutionMax());
292}
293
294auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
295 Matcher<int> fps_range_matcher;
296
297 if (last_frame_pixels <= 320 * 240) {
298 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 15:50:00299 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 12:08:39300 fps_range_matcher = AllOf(Ge(10), Le(15));
301 } else if (last_frame_pixels <= 640 * 480) {
302 fps_range_matcher = Ge(15);
303 } else {
304 fps_range_matcher = Eq(kDefaultFramerate);
305 }
306 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
307 fps_range_matcher);
308}
309
Evan Shrubsole5fd40602020-05-25 14:19:54310auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
311 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
312 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
313}
314
315auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
316 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
317}
318
319auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
320 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
321}
322
323auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
324 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
325 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
326}
327
328auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
329 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
330 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
331}
332
333auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
336}
337
338auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
341}
342
mflodmancc3d4422017-08-03 15:27:51343class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 18:45:46344 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23345 VideoStreamEncoderUnderTest(TimeController* time_controller,
346 TaskQueueFactory* task_queue_factory,
347 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 09:26:03348 const VideoStreamEncoderSettings& settings,
349 VideoStreamEncoder::BitrateAllocationCallbackType
350 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 12:31:23351 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 17:30:41352 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 13:03:05353 stats_proxy,
354 settings,
Yves Gerey665174f2018-06-19 13:03:05355 std::unique_ptr<OveruseFrameDetector>(
356 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 10:50:20357 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 09:26:03358 task_queue_factory,
359 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 12:31:23360 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 15:59:05361 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 11:02:36362 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 09:47:10363 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 14:29:22364 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 09:36:55365 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 14:29:22366 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 11:02:36367 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 15:26:01368 }
perkj803d97f2016-11-01 18:45:46369
Henrik Boström381d1092020-05-12 16:49:07370 void SetSourceAndWaitForRestrictionsUpdated(
371 rtc::VideoSourceInterface<VideoFrame>* source,
372 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 11:02:36373 FakeVideoSourceRestrictionsListener listener;
374 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07375 SetSource(source, degradation_preference);
376 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 11:02:36377 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07378 }
379
380 void SetSourceAndWaitForFramerateUpdated(
381 rtc::VideoSourceInterface<VideoFrame>* source,
382 const DegradationPreference& degradation_preference) {
383 overuse_detector_proxy_->framerate_updated_event()->Reset();
384 SetSource(source, degradation_preference);
385 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
386 }
387
388 void OnBitrateUpdatedAndWaitForManagedResources(
389 DataRate target_bitrate,
390 DataRate stable_target_bitrate,
391 DataRate link_allocation,
392 uint8_t fraction_lost,
393 int64_t round_trip_time_ms,
394 double cwnd_reduce_ratio) {
395 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
396 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
397 // Bitrate is updated on the encoder queue.
398 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 16:49:07399 }
400
kthelgason2fc52542017-03-03 08:24:41401 // This is used as a synchronisation mechanism, to make sure that the
402 // encoder queue is not blocked before we start sending it frames.
403 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 07:43:50404 rtc::Event event;
Yves Gerey665174f2018-06-19 13:03:05405 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 08:24:41406 ASSERT_TRUE(event.Wait(5000));
407 }
408
Henrik Boström91aa7322020-04-28 10:24:33409 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 13:25:32410 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 10:24:33411 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36413 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 12:31:23417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 10:24:33418 }
Tomas Gunnarsson612445e2020-09-21 12:31:23419
Henrik Boström91aa7322020-04-28 10:24:33420 void TriggerCpuUnderuse() {
421 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12422 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36423 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33424 event.Set();
425 });
426 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 12:31:23427 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 13:25:32428 }
kthelgason876222f2016-11-29 09:44:11429
Henrik Boström91aa7322020-04-28 10:24:33430 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 13:25:32431 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 10:24:33432 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12433 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36434 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33435 event.Set();
436 });
437 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 12:31:23438 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 13:25:32439 }
Åsa Perssonb67c44c2019-09-24 13:25:32440 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 10:24:33441 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12442 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36443 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33444 event.Set();
445 });
446 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 12:31:23447 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 10:24:33448 }
449
Tomas Gunnarsson612445e2020-09-21 12:31:23450 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 09:27:48451 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 14:29:22452 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
453 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 11:02:36454 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 18:45:46455};
456
Noah Richards51db4212019-06-12 13:59:12457// Simulates simulcast behavior and makes highest stream resolutions divisible
458// by 4.
459class CroppingVideoStreamFactory
460 : public VideoEncoderConfig::VideoStreamFactoryInterface {
461 public:
Åsa Persson17b29b92020-10-17 10:57:58462 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 13:59:12463
464 private:
465 std::vector<VideoStream> CreateEncoderStreams(
466 int width,
467 int height,
468 const VideoEncoderConfig& encoder_config) override {
469 std::vector<VideoStream> streams = test::CreateVideoStreams(
470 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 13:59:12471 return streams;
472 }
Noah Richards51db4212019-06-12 13:59:12473};
474
sprangb1ca0732017-02-01 16:38:12475class AdaptingFrameForwarder : public test::FrameForwarder {
476 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23477 explicit AdaptingFrameForwarder(TimeController* time_controller)
478 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 12:51:49479 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 16:38:12480
481 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 11:13:32482 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12483 adaptation_enabled_ = enabled;
484 }
485
asaperssonfab67072017-04-04 12:51:49486 bool adaption_enabled() const {
Markus Handella3765182020-07-08 11:13:32487 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12488 return adaptation_enabled_;
489 }
490
Henrik Boström1124ed12021-02-25 09:30:39491 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
492 // the resolution or frame rate was different than it is currently. If
493 // something else is modified, such as encoder resolutions, but the resolution
494 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-16 06:40:18495 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 11:13:32496 MutexLock lock(&mutex_);
asapersson09f05612017-05-16 06:40:18497 return last_wants_;
498 }
499
Danil Chapovalovb9b146c2018-06-15 10:28:07500 absl::optional<int> last_sent_width() const { return last_width_; }
501 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-09 01:04:29502
sprangb1ca0732017-02-01 16:38:12503 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 12:31:23504 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
505 time_controller_->AdvanceTime(TimeDelta::Millis(0));
506
sprangb1ca0732017-02-01 16:38:12507 int cropped_width = 0;
508 int cropped_height = 0;
509 int out_width = 0;
510 int out_height = 0;
sprangc5d62e22017-04-03 06:53:04511 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 12:31:23512 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
513 << "w=" << video_frame.width()
514 << "h=" << video_frame.height();
sprangc5d62e22017-04-03 06:53:04515 if (adapter_.AdaptFrameResolution(
516 video_frame.width(), video_frame.height(),
517 video_frame.timestamp_us() * 1000, &cropped_width,
518 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 22:49:37519 VideoFrame adapted_frame =
520 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43521 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 22:49:37522 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 17:05:30523 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 22:49:37524 .set_timestamp_ms(99)
525 .set_rotation(kVideoRotation_0)
526 .build();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:17527 if (video_frame.has_update_rect()) {
528 adapted_frame.set_update_rect(
529 video_frame.update_rect().ScaleWithFrame(
530 video_frame.width(), video_frame.height(), 0, 0,
531 video_frame.width(), video_frame.height(), out_width,
532 out_height));
533 }
sprangc5d62e22017-04-03 06:53:04534 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-09 01:04:29535 last_width_.emplace(adapted_frame.width());
536 last_height_.emplace(adapted_frame.height());
537 } else {
Danil Chapovalovb9b146c2018-06-15 10:28:07538 last_width_ = absl::nullopt;
539 last_height_ = absl::nullopt;
sprangc5d62e22017-04-03 06:53:04540 }
sprangb1ca0732017-02-01 16:38:12541 } else {
Tomas Gunnarsson612445e2020-09-21 12:31:23542 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 16:38:12543 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-09 01:04:29544 last_width_.emplace(video_frame.width());
545 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 16:38:12546 }
547 }
548
Åsa Perssonf5f7e8e2021-06-09 20:55:38549 void OnOutputFormatRequest(int width, int height) {
550 absl::optional<std::pair<int, int>> target_aspect_ratio =
551 std::make_pair(width, height);
552 absl::optional<int> max_pixel_count = width * height;
553 absl::optional<int> max_fps;
554 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
555 max_fps);
556 }
557
sprangb1ca0732017-02-01 16:38:12558 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
559 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 11:13:32560 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 09:30:39561 rtc::VideoSinkWants prev_wants = sink_wants_locked();
562 bool did_adapt =
563 prev_wants.max_pixel_count != wants.max_pixel_count ||
564 prev_wants.target_pixel_count != wants.target_pixel_count ||
565 prev_wants.max_framerate_fps != wants.max_framerate_fps;
566 if (did_adapt) {
567 last_wants_ = prev_wants;
568 }
Rasmus Brandt287e4642019-11-15 15:56:01569 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 06:37:30570 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 16:38:12571 }
Tomas Gunnarsson612445e2020-09-21 12:31:23572
573 TimeController* const time_controller_;
sprangb1ca0732017-02-01 16:38:12574 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 11:13:32575 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
576 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 10:28:07577 absl::optional<int> last_width_;
578 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 16:38:12579};
sprangc5d62e22017-04-03 06:53:04580
Niels Möller213618e2018-07-24 07:29:58581// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-03 06:53:04582class MockableSendStatisticsProxy : public SendStatisticsProxy {
583 public:
584 MockableSendStatisticsProxy(Clock* clock,
585 const VideoSendStream::Config& config,
586 VideoEncoderConfig::ContentType content_type)
587 : SendStatisticsProxy(clock, config, content_type) {}
588
589 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 11:13:32590 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04591 if (mock_stats_)
592 return *mock_stats_;
593 return SendStatisticsProxy::GetStats();
594 }
595
Niels Möller213618e2018-07-24 07:29:58596 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 11:13:32597 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 07:29:58598 if (mock_stats_)
599 return mock_stats_->input_frame_rate;
600 return SendStatisticsProxy::GetInputFrameRate();
601 }
sprangc5d62e22017-04-03 06:53:04602 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 11:13:32603 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04604 mock_stats_.emplace(stats);
605 }
606
607 void ResetMockStats() {
Markus Handella3765182020-07-08 11:13:32608 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04609 mock_stats_.reset();
610 }
611
Tomas Gunnarsson612445e2020-09-21 12:31:23612 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
613 on_frame_dropped_ = std::move(callback);
614 }
615
sprangc5d62e22017-04-03 06:53:04616 private:
Tomas Gunnarsson612445e2020-09-21 12:31:23617 void OnFrameDropped(DropReason reason) override {
618 SendStatisticsProxy::OnFrameDropped(reason);
619 if (on_frame_dropped_)
620 on_frame_dropped_(reason);
621 }
622
Markus Handella3765182020-07-08 11:13:32623 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 10:28:07624 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 12:31:23625 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-03 06:53:04626};
627
philipel9b058032020-02-10 10:30:00628class MockEncoderSelector
629 : public VideoEncoderFactory::EncoderSelectorInterface {
630 public:
Danil Chapovalov91fdc602020-05-14 17:17:51631 MOCK_METHOD(void,
632 OnCurrentEncoder,
633 (const SdpVideoFormat& format),
634 (override));
635 MOCK_METHOD(absl::optional<SdpVideoFormat>,
636 OnAvailableBitrate,
637 (const DataRate& rate),
638 (override));
639 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 10:30:00640};
641
perkj803d97f2016-11-01 18:45:46642} // namespace
643
mflodmancc3d4422017-08-03 15:27:51644class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 08:17:40645 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23646 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 08:17:40647
mflodmancc3d4422017-08-03 15:27:51648 VideoStreamEncoderTest()
perkj26091b12016-09-01 08:17:40649 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-03 06:45:26650 codec_width_(320),
651 codec_height_(240),
Åsa Persson8c1bf952018-09-13 08:42:19652 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 12:31:23653 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 07:04:13654 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-03 06:53:04655 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 12:31:23656 time_controller_.GetClock(),
perkj803d97f2016-11-01 18:45:46657 video_send_config_,
658 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 12:31:23659 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 08:17:40660
661 void SetUp() override {
perkj803d97f2016-11-01 18:45:46662 metrics::Reset();
perkj26091b12016-09-01 08:17:40663 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 07:04:13664 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56665 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 12:18:34666 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 13:36:51667 video_send_config_.rtp.payload_name = "FAKE";
668 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 08:17:40669
Per512ecb32016-09-23 13:52:06670 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51671 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 06:57:51672 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
673 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
674 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 15:41:30675 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 14:06:52676
Niels Möllerf1338562018-04-26 07:51:47677 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 12:37:00678 }
679
Per Kjellanderb03b6c82021-01-03 09:26:03680 void ConfigureEncoder(
681 VideoEncoderConfig video_encoder_config,
682 VideoStreamEncoder::BitrateAllocationCallbackType
683 allocation_callback_type =
684 VideoStreamEncoder::BitrateAllocationCallbackType::
685 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 15:27:51686 if (video_stream_encoder_)
687 video_stream_encoder_->Stop();
688 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 12:31:23689 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 09:26:03690 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 15:27:51691 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
692 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:41693 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 15:27:51694 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
695 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:47696 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:51697 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 12:37:00698 }
699
Per Kjellanderb03b6c82021-01-03 09:26:03700 void ResetEncoder(const std::string& payload_name,
701 size_t num_streams,
702 size_t num_temporal_layers,
703 unsigned char num_spatial_layers,
704 bool screenshare,
705 VideoStreamEncoder::BitrateAllocationCallbackType
706 allocation_callback_type =
707 VideoStreamEncoder::BitrateAllocationCallbackType::
708 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 13:36:51709 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 12:37:00710
711 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:51712 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
713 num_streams, &video_encoder_config);
714 for (auto& layer : video_encoder_config.simulcast_layers) {
715 layer.num_temporal_layers = num_temporal_layers;
716 layer.max_framerate = kDefaultFramerate;
717 }
Erik Språngd7329ca2019-02-21 20:19:53718 video_encoder_config.max_bitrate_bps =
719 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 14:06:52720 video_encoder_config.content_type =
721 screenshare ? VideoEncoderConfig::ContentType::kScreen
722 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 07:28:40723 if (payload_name == "VP9") {
724 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
725 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 13:29:23726 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 07:28:40727 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:43728 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
729 vp9_settings);
emircanbbcc3562017-08-18 07:28:40730 }
Per Kjellanderb03b6c82021-01-03 09:26:03731 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 08:17:40732 }
733
sprang57c2fff2017-01-16 14:24:02734 VideoFrame CreateFrame(int64_t ntp_time_ms,
735 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 17:05:30736 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43737 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 17:05:30738 destruction_event, codec_width_, codec_height_))
739 .set_ntp_time_ms(ntp_time_ms)
740 .set_timestamp_ms(99)
741 .set_rotation(kVideoRotation_0)
742 .build();
perkj26091b12016-09-01 08:17:40743 }
744
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26745 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
746 rtc::Event* destruction_event,
747 int offset_x) const {
Åsa Persson90719572021-04-08 17:05:30748 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43749 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 17:05:30750 destruction_event, codec_width_, codec_height_))
751 .set_ntp_time_ms(ntp_time_ms)
752 .set_timestamp_ms(99)
753 .set_rotation(kVideoRotation_0)
754 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
755 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26756 }
757
sprang57c2fff2017-01-16 14:24:02758 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 12:02:13759 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
760 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 17:05:30761 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 12:02:13762 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 17:05:30763 .set_ntp_time_ms(ntp_time_ms)
764 .set_timestamp_ms(ntp_time_ms)
765 .set_rotation(kVideoRotation_0)
766 .build();
perkj803d97f2016-11-01 18:45:46767 }
768
Evan Shrubsole895556e2020-10-05 07:15:13769 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 17:05:30770 return VideoFrame::Builder()
771 .set_video_frame_buffer(NV12Buffer::Create(width, height))
772 .set_ntp_time_ms(ntp_time_ms)
773 .set_timestamp_ms(ntp_time_ms)
774 .set_rotation(kVideoRotation_0)
775 .build();
Evan Shrubsole895556e2020-10-05 07:15:13776 }
777
Noah Richards51db4212019-06-12 13:59:12778 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
779 rtc::Event* destruction_event,
780 int width,
781 int height) const {
Åsa Persson90719572021-04-08 17:05:30782 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43783 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 17:05:30784 destruction_event, width, height))
785 .set_ntp_time_ms(ntp_time_ms)
786 .set_timestamp_ms(99)
787 .set_rotation(kVideoRotation_0)
788 .build();
Noah Richards51db4212019-06-12 13:59:12789 }
790
Evan Shrubsole895556e2020-10-05 07:15:13791 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
792 rtc::Event* destruction_event,
793 int width,
794 int height) const {
Åsa Persson90719572021-04-08 17:05:30795 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 17:21:43796 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 17:05:30797 destruction_event, width, height))
798 .set_ntp_time_ms(ntp_time_ms)
799 .set_timestamp_ms(99)
800 .set_rotation(kVideoRotation_0)
801 .build();
Evan Shrubsole895556e2020-10-05 07:15:13802 }
803
Noah Richards51db4212019-06-12 13:59:12804 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
805 rtc::Event* destruction_event) const {
806 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
807 codec_height_);
808 }
809
Åsa Perssonc29cb2c2019-03-25 11:06:59810 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 16:49:07811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:07812 DataRate::BitsPerSec(kTargetBitrateBps),
813 DataRate::BitsPerSec(kTargetBitrateBps),
814 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 11:06:59815
816 video_source_.IncomingCapturedFrame(
817 CreateFrame(1, codec_width_, codec_height_));
818 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 13:09:05819 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 11:06:59820 }
821
sprang4847ae62017-06-27 14:06:52822 void WaitForEncodedFrame(int64_t expected_ntp_time) {
823 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 12:31:23824 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52825 }
826
827 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
828 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:23829 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52830 return ok;
831 }
832
833 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
834 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 12:31:23835 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52836 }
837
838 void ExpectDroppedFrame() {
839 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 12:31:23840 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52841 }
842
843 bool WaitForFrame(int64_t timeout_ms) {
844 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:23845 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52846 return ok;
847 }
848
perkj26091b12016-09-01 08:17:40849 class TestEncoder : public test::FakeEncoder {
850 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23851 explicit TestEncoder(TimeController* time_controller)
852 : FakeEncoder(time_controller->GetClock()),
853 time_controller_(time_controller) {
854 RTC_DCHECK(time_controller_);
855 }
perkj26091b12016-09-01 08:17:40856
asaperssonfab67072017-04-04 12:51:49857 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 11:13:32858 MutexLock lock(&mutex_);
perkjfa10b552016-10-03 06:45:26859 return config_;
860 }
861
862 void BlockNextEncode() {
Markus Handella3765182020-07-08 11:13:32863 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-03 06:45:26864 block_next_encode_ = true;
865 }
866
Erik Språngaed30702018-11-05 11:57:17867 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 11:13:32868 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 15:44:42869 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 14:52:33870 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 11:42:18871 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 14:56:00872 info.scaling_settings = VideoEncoder::ScalingSettings(
873 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 11:42:18874 }
875 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 11:06:59876 for (int i = 0; i < kMaxSpatialLayers; ++i) {
877 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 14:27:35878 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 11:06:59879 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 14:27:35880 for (int tid = 0; tid < num_layers; ++tid)
881 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 11:06:59882 }
883 }
Erik Språngaed30702018-11-05 11:57:17884 }
Sergey Silkin6456e352019-07-08 15:56:40885
886 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 08:47:11887 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 15:50:00888 info.apply_alignment_to_all_simulcast_layers =
889 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 12:56:45890 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 11:57:17891 return info;
kthelgason876222f2016-11-29 09:44:11892 }
893
Erik Språngb7cb7b52019-02-26 14:52:33894 int32_t RegisterEncodeCompleteCallback(
895 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 11:13:32896 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:33897 encoded_image_callback_ = callback;
898 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
899 }
900
perkjfa10b552016-10-03 06:45:26901 void ContinueEncode() { continue_encode_event_.Set(); }
902
903 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
904 uint32_t timestamp) const {
Markus Handella3765182020-07-08 11:13:32905 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-03 06:45:26906 EXPECT_EQ(timestamp_, timestamp);
907 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
908 }
909
kthelgason2fc52542017-03-03 08:24:41910 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 11:13:32911 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 08:24:41912 quality_scaling_ = b;
913 }
kthelgasonad9010c2017-02-14 08:46:51914
Rasmus Brandt5cad55b2019-12-19 08:47:11915 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 11:13:32916 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 08:47:11917 requested_resolution_alignment_ = requested_resolution_alignment;
918 }
919
Åsa Perssonc5a74ff2020-09-20 15:50:00920 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
921 MutexLock lock(&local_mutex_);
922 apply_alignment_to_all_simulcast_layers_ = b;
923 }
924
Mirta Dvornicicccc1b572019-01-15 11:42:18925 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 11:13:32926 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 11:42:18927 is_hardware_accelerated_ = is_hardware_accelerated;
928 }
929
Åsa Perssonc29cb2c2019-03-25 11:06:59930 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
931 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 11:13:32932 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:59933 temporal_layers_supported_[spatial_idx] = supported;
934 }
935
Sergey Silkin6456e352019-07-08 15:56:40936 void SetResolutionBitrateLimits(
937 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 11:13:32938 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 15:56:40939 resolution_bitrate_limits_ = thresholds;
940 }
941
sprangfe627f32017-03-29 15:24:59942 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 11:13:32943 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 15:24:59944 force_init_encode_failed_ = force_failure;
945 }
946
Niels Möller6bb5ab92019-01-11 10:11:10947 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 11:13:32948 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:10949 rate_factor_ = rate_factor;
950 }
951
Erik Språngd7329ca2019-02-21 20:19:53952 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 11:13:32953 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:10954 return last_framerate_;
955 }
956
Erik Språngd7329ca2019-02-21 20:19:53957 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 11:13:32958 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26959 return last_update_rect_;
960 }
961
Niels Möller87e2d782019-03-07 09:18:23962 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 11:13:32963 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:53964 return last_frame_types_;
965 }
966
967 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 09:18:23968 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 14:43:58969 keyframe ? VideoFrameType::kVideoFrameKey
970 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:53971 {
Markus Handella3765182020-07-08 11:13:32972 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:53973 last_frame_types_ = frame_type;
974 }
Niels Möllerb859b322019-03-07 11:40:01975 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 20:19:53976 }
977
Sergey Silkin0e42cf72021-03-15 09:12:57978 void InjectEncodedImage(const EncodedImage& image,
979 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 11:13:32980 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 09:12:57981 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 14:52:33982 }
983
Mirta Dvornicic97910da2020-07-14 13:29:23984 void SetEncodedImageData(
985 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 11:13:32986 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 13:29:23987 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 14:30:16988 }
989
Erik Språngd7329ca2019-02-21 20:19:53990 void ExpectNullFrame() {
Markus Handella3765182020-07-08 11:13:32991 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:53992 expect_null_frame_ = true;
993 }
994
Erik Språng5056af02019-09-02 13:53:11995 absl::optional<VideoEncoder::RateControlParameters>
996 GetAndResetLastRateControlSettings() {
997 auto settings = last_rate_control_settings_;
998 last_rate_control_settings_.reset();
999 return settings;
Erik Språngd7329ca2019-02-21 20:19:531000 }
1001
Henrik Boström56db9ff2021-03-24 08:06:451002 int GetLastInputWidth() const {
1003 MutexLock lock(&local_mutex_);
1004 return last_input_width_;
1005 }
1006
1007 int GetLastInputHeight() const {
1008 MutexLock lock(&local_mutex_);
1009 return last_input_height_;
1010 }
1011
Evan Shrubsole895556e2020-10-05 07:15:131012 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1013 MutexLock lock(&local_mutex_);
1014 return last_input_pixel_format_;
1015 }
1016
Sergey Silkin5ee69672019-07-02 12:18:341017 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 11:13:321018 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341019 return num_encoder_initializations_;
1020 }
1021
Evan Shrubsole7c079f62019-09-26 07:55:031022 int GetNumSetRates() const {
Markus Handella3765182020-07-08 11:13:321023 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031024 return num_set_rates_;
1025 }
1026
Åsa Perssonc5a74ff2020-09-20 15:50:001027 VideoCodec video_codec() const {
1028 MutexLock lock(&local_mutex_);
1029 return video_codec_;
1030 }
1031
Evan Shrubsoleb556b082020-10-08 12:56:451032 void SetPreferredPixelFormats(
1033 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1034 pixel_formats) {
1035 MutexLock lock(&local_mutex_);
1036 preferred_pixel_formats_ = std::move(pixel_formats);
1037 }
1038
perkjfa10b552016-10-03 06:45:261039 private:
perkj26091b12016-09-01 08:17:401040 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 09:18:231041 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 08:17:401042 bool block_encode;
1043 {
Markus Handella3765182020-07-08 11:13:321044 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531045 if (expect_null_frame_) {
1046 EXPECT_EQ(input_image.timestamp(), 0u);
1047 EXPECT_EQ(input_image.width(), 1);
1048 last_frame_types_ = *frame_types;
1049 expect_null_frame_ = false;
1050 } else {
1051 EXPECT_GT(input_image.timestamp(), timestamp_);
1052 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1053 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1054 }
perkj26091b12016-09-01 08:17:401055
1056 timestamp_ = input_image.timestamp();
1057 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 18:45:461058 last_input_width_ = input_image.width();
1059 last_input_height_ = input_image.height();
perkj26091b12016-09-01 08:17:401060 block_encode = block_next_encode_;
1061 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:261062 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 20:19:531063 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 07:15:131064 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 08:17:401065 }
Niels Möllerb859b322019-03-07 11:40:011066 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 08:17:401067 if (block_encode)
perkja49cbd32016-09-16 14:53:411068 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 12:31:231069
perkj26091b12016-09-01 08:17:401070 return result;
1071 }
1072
Niels Möller08ae7ce2020-09-23 13:58:121073 CodecSpecificInfo EncodeHook(
1074 EncodedImage& encoded_image,
1075 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 15:30:361076 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231077 {
1078 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 15:30:361079 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 13:29:231080 }
1081 MutexLock lock(&local_mutex_);
1082 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 15:30:361083 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 13:29:231084 }
Danil Chapovalov2549f172020-08-12 15:30:361085 return codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231086 }
1087
sprangfe627f32017-03-29 15:24:591088 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 12:57:571089 const Settings& settings) override {
1090 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 12:18:341091
Markus Handella3765182020-07-08 11:13:321092 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331093 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 12:18:341094
1095 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 15:50:001096 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 12:18:341097
Erik Språng82fad3d2018-03-21 08:57:231098 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 15:24:591099 // Simulate setting up temporal layers, in order to validate the life
1100 // cycle of these objects.
Elad Aloncde8ab22019-03-20 10:56:201101 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 09:20:091102 frame_buffer_controller_ =
1103 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 15:24:591104 }
Erik Språngb7cb7b52019-02-26 14:52:331105 if (force_init_encode_failed_) {
1106 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 15:24:591107 return -1;
Erik Språngb7cb7b52019-02-26 14:52:331108 }
Mirta Dvornicicccc1b572019-01-15 11:42:181109
Erik Språngb7cb7b52019-02-26 14:52:331110 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 15:24:591111 return res;
1112 }
1113
Erik Språngb7cb7b52019-02-26 14:52:331114 int32_t Release() override {
Markus Handella3765182020-07-08 11:13:321115 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331116 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1117 initialized_ = EncoderState::kUninitialized;
1118 return FakeEncoder::Release();
1119 }
1120
Erik Språng16cb8f52019-04-12 11:59:091121 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 11:13:321122 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031123 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 10:11:101124 VideoBitrateAllocation adjusted_rate_allocation;
1125 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1126 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 11:59:091127 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 10:11:101128 adjusted_rate_allocation.SetBitrate(
1129 si, ti,
Erik Språng16cb8f52019-04-12 11:59:091130 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 10:11:101131 rate_factor_));
1132 }
1133 }
1134 }
Erik Språng16cb8f52019-04-12 11:59:091135 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 13:53:111136 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 11:59:091137 RateControlParameters adjusted_paramters = parameters;
1138 adjusted_paramters.bitrate = adjusted_rate_allocation;
1139 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 10:11:101140 }
1141
Tomas Gunnarsson612445e2020-09-21 12:31:231142 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321143 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 14:52:331144 enum class EncoderState {
1145 kUninitialized,
1146 kInitializationFailed,
1147 kInitialized
Markus Handella3765182020-07-08 11:13:321148 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1149 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 08:17:401150 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 11:13:321151 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1152 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1153 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1154 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1155 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1156 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 15:50:001157 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1158 false;
Markus Handella3765182020-07-08 11:13:321159 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 13:29:231160 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1161 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 10:56:201162 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 11:13:321163 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:591164 absl::optional<bool>
1165 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 11:13:321166 local_mutex_);
1167 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1168 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1169 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 13:53:111170 absl::optional<VideoEncoder::RateControlParameters>
1171 last_rate_control_settings_;
Markus Handella3765182020-07-08 11:13:321172 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1173 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 09:18:231174 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 20:19:531175 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 11:13:321176 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1177 nullptr;
Evan Shrubsolefb862742020-03-16 15:18:361178 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 11:13:321179 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 15:56:401180 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 11:13:321181 RTC_GUARDED_BY(local_mutex_);
1182 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 15:50:001183 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 07:15:131184 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1185 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 12:56:451186 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1187 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 08:17:401188 };
1189
mflodmancc3d4422017-08-03 15:27:511190 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 08:17:401191 public:
Tomas Gunnarsson612445e2020-09-21 12:31:231192 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1193 : time_controller_(time_controller), test_encoder_(test_encoder) {
1194 RTC_DCHECK(time_controller_);
1195 }
perkj26091b12016-09-01 08:17:401196
perkj26091b12016-09-01 08:17:401197 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 14:06:521198 EXPECT_TRUE(
1199 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1200 }
1201
1202 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1203 int64_t timeout_ms) {
perkj26091b12016-09-01 08:17:401204 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:231205 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 14:06:521206 return false;
perkj26091b12016-09-01 08:17:401207 {
Markus Handella3765182020-07-08 11:13:321208 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121209 timestamp = last_timestamp_;
perkj26091b12016-09-01 08:17:401210 }
1211 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 14:06:521212 return true;
perkj26091b12016-09-01 08:17:401213 }
1214
sprangb1ca0732017-02-01 16:38:121215 void WaitForEncodedFrame(uint32_t expected_width,
1216 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 12:31:231217 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 13:13:561218 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-03 06:53:041219 }
1220
Åsa Perssonc74d8da2017-12-04 13:13:561221 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-03 06:53:041222 uint32_t expected_height) {
sprangb1ca0732017-02-01 16:38:121223 uint32_t width = 0;
1224 uint32_t height = 0;
sprangb1ca0732017-02-01 16:38:121225 {
Markus Handella3765182020-07-08 11:13:321226 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121227 width = last_width_;
1228 height = last_height_;
1229 }
1230 EXPECT_EQ(expected_height, height);
1231 EXPECT_EQ(expected_width, width);
1232 }
1233
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141234 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1235 VideoRotation rotation;
1236 {
Markus Handella3765182020-07-08 11:13:321237 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141238 rotation = last_rotation_;
1239 }
1240 EXPECT_EQ(expected_rotation, rotation);
1241 }
1242
Tomas Gunnarsson612445e2020-09-21 12:31:231243 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 15:02:221244
sprangc5d62e22017-04-03 06:53:041245 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 12:31:231246 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1247 bool ret = encoded_frame_event_.Wait(timeout_ms);
1248 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1249 return ret;
sprangc5d62e22017-04-03 06:53:041250 }
1251
perkj26091b12016-09-01 08:17:401252 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 11:13:321253 MutexLock lock(&mutex_);
perkj26091b12016-09-01 08:17:401254 expect_frames_ = false;
1255 }
1256
asaperssonfab67072017-04-04 12:51:491257 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 11:13:321258 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061259 return number_of_reconfigurations_;
1260 }
1261
asaperssonfab67072017-04-04 12:51:491262 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 11:13:321263 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061264 return min_transmit_bitrate_bps_;
1265 }
1266
Erik Språngd7329ca2019-02-21 20:19:531267 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 11:13:321268 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 20:19:531269 num_expected_layers_ = num_layers;
1270 }
1271
Erik Språngb7cb7b52019-02-26 14:52:331272 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 11:13:321273 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331274 return last_capture_time_ms_;
1275 }
1276
Sergey Silkin0e42cf72021-03-15 09:12:571277 const EncodedImage& GetLastEncodedImage() {
1278 MutexLock lock(&mutex_);
1279 return last_encoded_image_;
1280 }
1281
Mirta Dvornicic28f0eb22019-05-28 14:30:161282 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 11:13:321283 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 14:30:161284 return std::move(last_encoded_image_data_);
1285 }
1286
Per Kjellanderdcef6412020-10-07 13:09:051287 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1288 MutexLock lock(&mutex_);
1289 return last_bitrate_allocation_;
1290 }
1291
1292 int number_of_bitrate_allocations() const {
1293 MutexLock lock(&mutex_);
1294 return number_of_bitrate_allocations_;
1295 }
1296
Per Kjellandera9434842020-10-15 15:53:221297 VideoLayersAllocation GetLastVideoLayersAllocation() {
1298 MutexLock lock(&mutex_);
1299 return last_layers_allocation_;
1300 }
1301
1302 int number_of_layers_allocations() const {
1303 MutexLock lock(&mutex_);
1304 return number_of_layers_allocations_;
1305 }
1306
perkj26091b12016-09-01 08:17:401307 private:
sergeyu2cb155a2016-11-04 18:39:291308 Result OnEncodedImage(
1309 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 15:30:361310 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 11:13:321311 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061312 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 09:12:571313 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 14:30:161314 last_encoded_image_data_ = std::vector<uint8_t>(
1315 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 20:19:531316 uint32_t timestamp = encoded_image.Timestamp();
1317 if (last_timestamp_ != timestamp) {
1318 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 12:02:131319 last_width_ = encoded_image._encodedWidth;
1320 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 20:19:531321 } else {
1322 ++num_received_layers_;
Erik Språng7444b192021-06-02 12:02:131323 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1324 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 20:19:531325 }
1326 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 14:52:331327 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141328 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 20:19:531329 if (num_received_layers_ == num_expected_layers_) {
1330 encoded_frame_event_.Set();
1331 }
sprangb1ca0732017-02-01 16:38:121332 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 13:52:061333 }
1334
Rasmus Brandtc402dbe2019-02-04 10:09:461335 void OnEncoderConfigurationChanged(
1336 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 12:10:271337 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 10:09:461338 VideoEncoderConfig::ContentType content_type,
1339 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 11:13:321340 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061341 ++number_of_reconfigurations_;
1342 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1343 }
1344
Per Kjellanderdcef6412020-10-07 13:09:051345 void OnBitrateAllocationUpdated(
1346 const VideoBitrateAllocation& allocation) override {
1347 MutexLock lock(&mutex_);
1348 ++number_of_bitrate_allocations_;
1349 last_bitrate_allocation_ = allocation;
1350 }
1351
Per Kjellandera9434842020-10-15 15:53:221352 void OnVideoLayersAllocationUpdated(
1353 VideoLayersAllocation allocation) override {
1354 MutexLock lock(&mutex_);
1355 ++number_of_layers_allocations_;
1356 last_layers_allocation_ = allocation;
1357 rtc::StringBuilder log;
1358 for (const auto& layer : allocation.active_spatial_layers) {
1359 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1360 << "[";
1361 for (const auto target_bitrate :
1362 layer.target_bitrate_per_temporal_layer) {
1363 log << target_bitrate.kbps() << ",";
1364 }
1365 log << "]";
1366 }
1367 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1368 }
1369
Tomas Gunnarsson612445e2020-09-21 12:31:231370 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321371 mutable Mutex mutex_;
perkj26091b12016-09-01 08:17:401372 TestEncoder* test_encoder_;
1373 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 09:12:571374 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 14:30:161375 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 16:38:121376 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 14:52:331377 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 16:38:121378 uint32_t last_height_ = 0;
1379 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141380 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 20:19:531381 size_t num_expected_layers_ = 1;
1382 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 08:17:401383 bool expect_frames_ = true;
Per512ecb32016-09-23 13:52:061384 int number_of_reconfigurations_ = 0;
1385 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 13:09:051386 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1387 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 15:53:221388 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1389 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 08:17:401390 };
1391
Sergey Silkin5ee69672019-07-02 12:18:341392 class VideoBitrateAllocatorProxyFactory
1393 : public VideoBitrateAllocatorFactory {
1394 public:
1395 VideoBitrateAllocatorProxyFactory()
1396 : bitrate_allocator_factory_(
1397 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1398
1399 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1400 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 11:13:321401 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341402 codec_config_ = codec;
1403 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1404 }
1405
1406 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 11:13:321407 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341408 return codec_config_;
1409 }
1410
1411 private:
1412 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1413
Markus Handella3765182020-07-08 11:13:321414 mutable Mutex mutex_;
1415 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341416 };
1417
Tomas Gunnarsson612445e2020-09-21 12:31:231418 Clock* clock() { return time_controller_.GetClock(); }
1419 void AdvanceTime(TimeDelta duration) {
1420 time_controller_.AdvanceTime(duration);
1421 }
1422
1423 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1424
1425 protected:
1426 virtual TaskQueueFactory* GetTaskQueueFactory() {
1427 return time_controller_.GetTaskQueueFactory();
1428 }
1429
1430 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 08:17:401431 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 15:41:301432 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 13:52:061433 int codec_width_;
1434 int codec_height_;
sprang4847ae62017-06-27 14:06:521435 int max_framerate_;
perkj26091b12016-09-01 08:17:401436 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 07:07:241437 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 12:18:341438 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-03 06:53:041439 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 08:17:401440 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 12:31:231441 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 15:27:511442 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 08:17:401443};
1444
mflodmancc3d4422017-08-03 15:27:511445TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 16:49:071446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071447 DataRate::BitsPerSec(kTargetBitrateBps),
1448 DataRate::BitsPerSec(kTargetBitrateBps),
1449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 07:43:501450 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411451 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 14:06:521452 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 14:53:411453 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 15:27:511454 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401455}
1456
mflodmancc3d4422017-08-03 15:27:511457TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 08:17:401458 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 07:43:501459 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 11:05:491460 // The encoder will cache up to one frame for a short duration. Adding two
1461 // frames means that the first frame will be dropped and the second frame will
1462 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 14:53:411463 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 12:31:231464 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 11:05:491465 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 14:53:411466 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401467
Henrik Boström381d1092020-05-12 16:49:071468 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071469 DataRate::BitsPerSec(kTargetBitrateBps),
1470 DataRate::BitsPerSec(kTargetBitrateBps),
1471 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401472
Sebastian Janssona3177052018-04-10 11:05:491473 // The pending frame should be received.
sprang4847ae62017-06-27 14:06:521474 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 11:05:491475 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1476
1477 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:511478 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401479}
1480
mflodmancc3d4422017-08-03 15:27:511481TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 16:49:071482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071483 DataRate::BitsPerSec(kTargetBitrateBps),
1484 DataRate::BitsPerSec(kTargetBitrateBps),
1485 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 14:53:411486 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521487 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401488
Henrik Boström381d1092020-05-12 16:49:071489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1490 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1491 0, 0, 0);
Sebastian Janssona3177052018-04-10 11:05:491492 // The encoder will cache up to one frame for a short duration. Adding two
1493 // frames means that the first frame will be dropped and the second frame will
1494 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 14:53:411495 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 11:05:491496 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401497
Henrik Boström381d1092020-05-12 16:49:071498 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071499 DataRate::BitsPerSec(kTargetBitrateBps),
1500 DataRate::BitsPerSec(kTargetBitrateBps),
1501 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:521502 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 11:05:491503 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1504 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 15:27:511505 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401506}
1507
mflodmancc3d4422017-08-03 15:27:511508TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 16:49:071509 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071510 DataRate::BitsPerSec(kTargetBitrateBps),
1511 DataRate::BitsPerSec(kTargetBitrateBps),
1512 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 14:53:411513 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521514 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401515
1516 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 14:53:411517 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 08:17:401518
perkja49cbd32016-09-16 14:53:411519 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521520 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511521 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401522}
1523
mflodmancc3d4422017-08-03 15:27:511524TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 16:49:071525 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071526 DataRate::BitsPerSec(kTargetBitrateBps),
1527 DataRate::BitsPerSec(kTargetBitrateBps),
1528 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401529
perkja49cbd32016-09-16 14:53:411530 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521531 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401532
mflodmancc3d4422017-08-03 15:27:511533 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401534 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 07:43:501535 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411536 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1537 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401538}
1539
Tomas Gunnarsson612445e2020-09-21 12:31:231540class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1541 public:
1542 VideoStreamEncoderBlockedTest() {}
1543
1544 TaskQueueFactory* GetTaskQueueFactory() override {
1545 return task_queue_factory_.get();
1546 }
1547
1548 private:
1549 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1550 CreateDefaultTaskQueueFactory();
1551};
1552
1553TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 16:49:071554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071555 DataRate::BitsPerSec(kTargetBitrateBps),
1556 DataRate::BitsPerSec(kTargetBitrateBps),
1557 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401558
Tomas Gunnarsson612445e2020-09-21 12:31:231559 int dropped_count = 0;
1560 stats_proxy_->SetDroppedFrameCallback(
1561 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1562 ++dropped_count;
1563 });
1564
perkj26091b12016-09-01 08:17:401565 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 14:53:411566 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521567 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401568 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1569 // call to ContinueEncode.
perkja49cbd32016-09-16 14:53:411570 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1571 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401572 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 14:06:521573 WaitForEncodedFrame(3);
perkj26091b12016-09-01 08:17:401574
mflodmancc3d4422017-08-03 15:27:511575 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 12:31:231576
1577 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 08:17:401578}
1579
Henrik Boström56db9ff2021-03-24 08:06:451580TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 16:49:071581 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071582 DataRate::BitsPerSec(kTargetBitrateBps),
1583 DataRate::BitsPerSec(kTargetBitrateBps),
1584 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121585
1586 rtc::Event frame_destroyed_event;
1587 video_source_.IncomingCapturedFrame(
1588 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 08:06:451589 WaitForEncodedFrame(1);
1590 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1591 fake_encoder_.GetLastInputPixelFormat());
1592 EXPECT_EQ(fake_encoder_.codec_config().width,
1593 fake_encoder_.GetLastInputWidth());
1594 EXPECT_EQ(fake_encoder_.codec_config().height,
1595 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 13:59:121596 video_stream_encoder_->Stop();
1597}
1598
Henrik Boström56db9ff2021-03-24 08:06:451599TEST_F(VideoStreamEncoderTest,
1600 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 13:59:121601 // Use the cropping factory.
1602 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:431603 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 13:59:121604 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1605 kMaxPayloadLength);
1606 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1607
1608 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 16:49:071609 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071610 DataRate::BitsPerSec(kTargetBitrateBps),
1611 DataRate::BitsPerSec(kTargetBitrateBps),
1612 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121613 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1614 WaitForEncodedFrame(1);
1615 // The encoder will have been configured once.
1616 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1617 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1618 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1619
1620 // Now send in a fake frame that needs to be cropped as the width/height
1621 // aren't divisible by 4 (see CreateEncoderStreams above).
1622 rtc::Event frame_destroyed_event;
1623 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1624 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 08:06:451625 WaitForEncodedFrame(2);
1626 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1627 fake_encoder_.GetLastInputPixelFormat());
1628 EXPECT_EQ(fake_encoder_.codec_config().width,
1629 fake_encoder_.GetLastInputWidth());
1630 EXPECT_EQ(fake_encoder_.codec_config().height,
1631 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 13:59:121632 video_stream_encoder_->Stop();
1633}
1634
Evan Shrubsole895556e2020-10-05 07:15:131635TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1637 DataRate::BitsPerSec(kTargetBitrateBps),
1638 DataRate::BitsPerSec(kTargetBitrateBps),
1639 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1640
1641 video_source_.IncomingCapturedFrame(
1642 CreateNV12Frame(1, codec_width_, codec_height_));
1643 WaitForEncodedFrame(1);
1644 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1645 fake_encoder_.GetLastInputPixelFormat());
1646 video_stream_encoder_->Stop();
1647}
1648
Henrik Boström56db9ff2021-03-24 08:06:451649TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 12:56:451650 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1651 DataRate::BitsPerSec(kTargetBitrateBps),
1652 DataRate::BitsPerSec(kTargetBitrateBps),
1653 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1654
1655 fake_encoder_.SetPreferredPixelFormats({});
1656
1657 rtc::Event frame_destroyed_event;
1658 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1659 1, &frame_destroyed_event, codec_width_, codec_height_));
1660 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451661 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451662 fake_encoder_.GetLastInputPixelFormat());
1663 video_stream_encoder_->Stop();
1664}
1665
Henrik Boström56db9ff2021-03-24 08:06:451666TEST_F(VideoStreamEncoderTest,
1667 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 12:56:451668 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1669 DataRate::BitsPerSec(kTargetBitrateBps),
1670 DataRate::BitsPerSec(kTargetBitrateBps),
1671 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1672
1673 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1674
1675 rtc::Event frame_destroyed_event;
1676 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1677 1, &frame_destroyed_event, codec_width_, codec_height_));
1678 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451679 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451680 fake_encoder_.GetLastInputPixelFormat());
1681 video_stream_encoder_->Stop();
1682}
1683
Henrik Boström56db9ff2021-03-24 08:06:451684TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 12:56:451685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1686 DataRate::BitsPerSec(kTargetBitrateBps),
1687 DataRate::BitsPerSec(kTargetBitrateBps),
1688 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1689
1690 // Fake NV12 native frame does not allow mapping to I444.
1691 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1692
1693 rtc::Event frame_destroyed_event;
1694 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1695 1, &frame_destroyed_event, codec_width_, codec_height_));
1696 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451697 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 12:56:451698 fake_encoder_.GetLastInputPixelFormat());
1699 video_stream_encoder_->Stop();
1700}
1701
Henrik Boström56db9ff2021-03-24 08:06:451702TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 07:15:131703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1704 DataRate::BitsPerSec(kTargetBitrateBps),
1705 DataRate::BitsPerSec(kTargetBitrateBps),
1706 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1707
1708 rtc::Event frame_destroyed_event;
1709 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1710 1, &frame_destroyed_event, codec_width_, codec_height_));
1711 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 08:06:451712 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 07:15:131713 fake_encoder_.GetLastInputPixelFormat());
1714 video_stream_encoder_->Stop();
1715}
1716
Ying Wang9b881ab2020-02-07 13:29:321717TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 16:49:071718 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071719 DataRate::BitsPerSec(kTargetBitrateBps),
1720 DataRate::BitsPerSec(kTargetBitrateBps),
1721 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 13:29:321722 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1723 WaitForEncodedFrame(1);
1724
Henrik Boström381d1092020-05-12 16:49:071725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071726 DataRate::BitsPerSec(kTargetBitrateBps),
1727 DataRate::BitsPerSec(kTargetBitrateBps),
1728 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 13:29:321729 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1730 // frames. Adding two frames means that the first frame will be dropped and
1731 // the second frame will be sent to the encoder.
1732 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1733 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1734 WaitForEncodedFrame(3);
1735 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1736 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1737 WaitForEncodedFrame(5);
1738 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1739 video_stream_encoder_->Stop();
1740}
1741
mflodmancc3d4422017-08-03 15:27:511742TEST_F(VideoStreamEncoderTest,
1743 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 16:49:071744 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071745 DataRate::BitsPerSec(kTargetBitrateBps),
1746 DataRate::BitsPerSec(kTargetBitrateBps),
1747 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 20:37:571748 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061749
1750 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551751 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521752 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571753 // The encoder will have been configured once when the first frame is
1754 // received.
1755 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061756
1757 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:511758 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 13:52:061759 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 15:27:511760 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:471761 kMaxPayloadLength);
Per512ecb32016-09-23 13:52:061762
1763 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551764 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521765 WaitForEncodedFrame(2);
Per21d45d22016-10-30 20:37:571766 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-30 06:25:401767 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-30 05:39:101768
mflodmancc3d4422017-08-03 15:27:511769 video_stream_encoder_->Stop();
perkj26105b42016-09-30 05:39:101770}
1771
mflodmancc3d4422017-08-03 15:27:511772TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 16:49:071773 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071774 DataRate::BitsPerSec(kTargetBitrateBps),
1775 DataRate::BitsPerSec(kTargetBitrateBps),
1776 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-03 06:45:261777
1778 // Capture a frame and wait for it to synchronize with the encoder thread.
1779 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521780 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571781 // The encoder will have been configured once.
1782 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261783 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1784 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1785
1786 codec_width_ *= 2;
1787 codec_height_ *= 2;
1788 // Capture a frame with a higher resolution and wait for it to synchronize
1789 // with the encoder thread.
1790 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521791 WaitForEncodedFrame(2);
perkjfa10b552016-10-03 06:45:261792 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1793 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 20:37:571794 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261795
mflodmancc3d4422017-08-03 15:27:511796 video_stream_encoder_->Stop();
perkjfa10b552016-10-03 06:45:261797}
1798
Sergey Silkin443b7ee2019-06-28 10:53:071799TEST_F(VideoStreamEncoderTest,
1800 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 16:49:071801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071802 DataRate::BitsPerSec(kTargetBitrateBps),
1803 DataRate::BitsPerSec(kTargetBitrateBps),
1804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 10:53:071805
1806 // Capture a frame and wait for it to synchronize with the encoder thread.
1807 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1808 WaitForEncodedFrame(1);
1809
1810 VideoEncoderConfig video_encoder_config;
1811 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1812 // Changing the max payload data length recreates encoder.
1813 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1814 kMaxPayloadLength / 2);
1815
1816 // Capture a frame and wait for it to synchronize with the encoder thread.
1817 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1818 WaitForEncodedFrame(2);
1819 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1820
1821 video_stream_encoder_->Stop();
1822}
1823
Sergey Silkin5ee69672019-07-02 12:18:341824TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 16:49:071825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071826 DataRate::BitsPerSec(kTargetBitrateBps),
1827 DataRate::BitsPerSec(kTargetBitrateBps),
1828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 12:18:341829
1830 VideoEncoderConfig video_encoder_config;
1831 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1832 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1833 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1834 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1835 kMaxPayloadLength);
1836
1837 // Capture a frame and wait for it to synchronize with the encoder thread.
1838 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1839 WaitForEncodedFrame(1);
1840 // The encoder will have been configured once when the first frame is
1841 // received.
1842 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1843 EXPECT_EQ(kTargetBitrateBps,
1844 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1845 EXPECT_EQ(kStartBitrateBps,
1846 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1847
Sergey Silkin6456e352019-07-08 15:56:401848 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1849 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 12:18:341850 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1851 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1852 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1853 kMaxPayloadLength);
1854
1855 // Capture a frame and wait for it to synchronize with the encoder thread.
1856 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1857 WaitForEncodedFrame(2);
1858 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1859 // Bitrate limits have changed - rate allocator should be reconfigured,
1860 // encoder should not be reconfigured.
1861 EXPECT_EQ(kTargetBitrateBps * 2,
1862 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1863 EXPECT_EQ(kStartBitrateBps * 2,
1864 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1865 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1866
1867 video_stream_encoder_->Stop();
1868}
1869
Sergey Silkin6456e352019-07-08 15:56:401870TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 13:48:401871 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 16:49:071872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071873 DataRate::BitsPerSec(kTargetBitrateBps),
1874 DataRate::BitsPerSec(kTargetBitrateBps),
1875 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401876
Sergey Silkincd02eba2020-01-20 13:48:401877 const uint32_t kMinEncBitrateKbps = 100;
1878 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051879 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 13:48:401880 /*frame_size_pixels=*/codec_width_ * codec_height_,
1881 /*min_start_bitrate_bps=*/0,
1882 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1883 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051884 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1885
Sergey Silkincd02eba2020-01-20 13:48:401886 VideoEncoderConfig video_encoder_config;
1887 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1888 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1889 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1890 (kMinEncBitrateKbps + 1) * 1000;
1891 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1892 kMaxPayloadLength);
1893
1894 // When both encoder and app provide bitrate limits, the intersection of
1895 // provided sets should be used.
1896 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1897 WaitForEncodedFrame(1);
1898 EXPECT_EQ(kMaxEncBitrateKbps,
1899 bitrate_allocator_factory_.codec_config().maxBitrate);
1900 EXPECT_EQ(kMinEncBitrateKbps + 1,
1901 bitrate_allocator_factory_.codec_config().minBitrate);
1902
1903 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1904 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1905 (kMinEncBitrateKbps - 1) * 1000;
1906 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1907 kMaxPayloadLength);
1908 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 14:04:051909 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 13:48:401910 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 14:04:051911 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:401912 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051913 bitrate_allocator_factory_.codec_config().minBitrate);
1914
Sergey Silkincd02eba2020-01-20 13:48:401915 video_stream_encoder_->Stop();
1916}
1917
1918TEST_F(VideoStreamEncoderTest,
1919 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 16:49:071920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071921 DataRate::BitsPerSec(kTargetBitrateBps),
1922 DataRate::BitsPerSec(kTargetBitrateBps),
1923 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 13:48:401924
1925 const uint32_t kMinAppBitrateKbps = 100;
1926 const uint32_t kMaxAppBitrateKbps = 200;
1927 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1928 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1929 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1930 /*frame_size_pixels=*/codec_width_ * codec_height_,
1931 /*min_start_bitrate_bps=*/0,
1932 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1933 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1934 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1935
1936 VideoEncoderConfig video_encoder_config;
1937 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1938 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1939 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1940 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051941 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1942 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 14:04:051943
Sergey Silkincd02eba2020-01-20 13:48:401944 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1945 WaitForEncodedFrame(1);
1946 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051947 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:401948 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051949 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 15:56:401950
1951 video_stream_encoder_->Stop();
1952}
1953
1954TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 14:04:051955 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 16:49:071956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071957 DataRate::BitsPerSec(kTargetBitrateBps),
1958 DataRate::BitsPerSec(kTargetBitrateBps),
1959 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401960
1961 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 14:04:051962 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 15:56:401963 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 14:04:051964 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 15:56:401965 fake_encoder_.SetResolutionBitrateLimits(
1966 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1967
1968 VideoEncoderConfig video_encoder_config;
1969 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1970 video_encoder_config.max_bitrate_bps = 0;
1971 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1972 kMaxPayloadLength);
1973
Sergey Silkin6b2cec12019-08-09 14:04:051974 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 15:56:401975 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1976 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 14:04:051977 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1978 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401979 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1980 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1981
Sergey Silkin6b2cec12019-08-09 14:04:051982 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 15:56:401983 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1984 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 14:04:051985 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1986 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401987 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1988 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1989
Sergey Silkin6b2cec12019-08-09 14:04:051990 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 15:56:401991 // encoder for 360p should be used.
1992 video_source_.IncomingCapturedFrame(
1993 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1994 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 14:04:051995 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1996 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401997 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1998 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1999
Sergey Silkin6b2cec12019-08-09 14:04:052000 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 15:56:402001 // ignored.
2002 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2003 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 14:04:052004 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2005 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402006 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2007 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:052008 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2009 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402010 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2011 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2012
2013 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2014 // for 270p should be used.
2015 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2016 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 14:04:052017 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2018 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:402019 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2020 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2021
2022 video_stream_encoder_->Stop();
2023}
2024
Sergey Silkin6b2cec12019-08-09 14:04:052025TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 16:49:072026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072027 DataRate::BitsPerSec(kTargetBitrateBps),
2028 DataRate::BitsPerSec(kTargetBitrateBps),
2029 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 14:04:052030
2031 VideoEncoderConfig video_encoder_config;
2032 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2033 video_encoder_config.max_bitrate_bps = 0;
2034 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2035 kMaxPayloadLength);
2036
2037 // Encode 720p frame to get the default encoder target bitrate.
2038 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2039 WaitForEncodedFrame(1);
2040 const uint32_t kDefaultTargetBitrateFor720pKbps =
2041 bitrate_allocator_factory_.codec_config()
2042 .simulcastStream[0]
2043 .targetBitrate;
2044
2045 // Set the max recommended encoder bitrate to something lower than the default
2046 // target bitrate.
2047 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2048 1280 * 720, 10 * 1000, 10 * 1000,
2049 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2050 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2051
2052 // Change resolution to trigger encoder reinitialization.
2053 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2054 WaitForEncodedFrame(2);
2055 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2056 WaitForEncodedFrame(3);
2057
2058 // Ensure the target bitrate is capped by the max bitrate.
2059 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2060 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2061 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2062 .simulcastStream[0]
2063 .targetBitrate *
2064 1000,
2065 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2066
2067 video_stream_encoder_->Stop();
2068}
2069
Åsa Perssona7e34d32021-01-20 14:36:132070TEST_F(VideoStreamEncoderTest,
2071 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2072 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2073 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2074 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2075 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2076 fake_encoder_.SetResolutionBitrateLimits(
2077 {kEncoderLimits270p, kEncoderLimits360p});
2078
2079 // Two streams, highest stream active.
2080 VideoEncoderConfig config;
2081 const int kNumStreams = 2;
2082 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2083 config.max_bitrate_bps = 0;
2084 config.simulcast_layers[0].active = false;
2085 config.simulcast_layers[1].active = true;
2086 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432087 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132088 "VP8", /*max qp*/ 56, /*screencast*/ false,
2089 /*screenshare enabled*/ false);
2090 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2091
2092 // The encoder bitrate limits for 270p should be used.
2093 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2094 EXPECT_FALSE(WaitForFrame(1000));
2095 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2096 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2097 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2098 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2099 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2100
2101 // The encoder bitrate limits for 360p should be used.
2102 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2103 EXPECT_FALSE(WaitForFrame(1000));
2104 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2105 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2106 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2107 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2108
2109 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2110 video_source_.IncomingCapturedFrame(
2111 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2112 EXPECT_FALSE(WaitForFrame(1000));
2113 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2114 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2115 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2116 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2117
2118 // Resolution higher than 360p. Encoder limits should be ignored.
2119 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2120 EXPECT_FALSE(WaitForFrame(1000));
2121 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2122 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2123 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2124 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2125 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2126 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2127 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2128 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2129
2130 // Resolution lower than 270p. The encoder limits for 270p should be used.
2131 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2132 EXPECT_FALSE(WaitForFrame(1000));
2133 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2134 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2135 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2136 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2137
2138 video_stream_encoder_->Stop();
2139}
2140
2141TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:512142 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2143 // Two streams, highest stream active.
2144 VideoEncoderConfig config;
2145 const int kNumStreams = 2;
2146 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2147 config.max_bitrate_bps = 0;
2148 config.simulcast_layers[0].active = false;
2149 config.simulcast_layers[1].active = true;
2150 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432151 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 09:39:512152 "VP8", /*max qp*/ 56, /*screencast*/ false,
2153 /*screenshare enabled*/ false);
2154 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2155
2156 // Default bitrate limits for 270p should be used.
2157 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2158 kDefaultLimits270p =
2159 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192160 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:512161 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2162 EXPECT_FALSE(WaitForFrame(1000));
2163 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2164 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2165 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2166 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2167 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2168
2169 // Default bitrate limits for 360p should be used.
2170 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2171 kDefaultLimits360p =
2172 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192173 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:512174 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2175 EXPECT_FALSE(WaitForFrame(1000));
2176 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2177 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2178 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2179 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2180
2181 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2182 video_source_.IncomingCapturedFrame(
2183 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2184 EXPECT_FALSE(WaitForFrame(1000));
2185 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2186 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2187 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2188 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2189
2190 // Default bitrate limits for 540p should be used.
2191 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2192 kDefaultLimits540p =
2193 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192194 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 09:39:512195 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2196 EXPECT_FALSE(WaitForFrame(1000));
2197 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2198 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2199 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2200 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2201
2202 video_stream_encoder_->Stop();
2203}
2204
2205TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:132206 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2207 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2208 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2209 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2210 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2211 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2212 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2213 fake_encoder_.SetResolutionBitrateLimits(
2214 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2215
2216 // Three streams, middle stream active.
2217 VideoEncoderConfig config;
2218 const int kNumStreams = 3;
2219 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2220 config.simulcast_layers[0].active = false;
2221 config.simulcast_layers[1].active = true;
2222 config.simulcast_layers[2].active = false;
2223 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432224 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132225 "VP8", /*max qp*/ 56, /*screencast*/ false,
2226 /*screenshare enabled*/ false);
2227 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2228
2229 // The encoder bitrate limits for 360p should be used.
2230 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2231 EXPECT_FALSE(WaitForFrame(1000));
2232 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2233 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2234 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2235 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2236 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2237
2238 // The encoder bitrate limits for 270p should be used.
2239 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2240 EXPECT_FALSE(WaitForFrame(1000));
2241 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2242 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2243 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2244 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2245
2246 video_stream_encoder_->Stop();
2247}
2248
2249TEST_F(VideoStreamEncoderTest,
2250 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2251 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2252 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2253 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2254 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2255 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2256 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2257 fake_encoder_.SetResolutionBitrateLimits(
2258 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2259
2260 // Three streams, lowest stream active.
2261 VideoEncoderConfig config;
2262 const int kNumStreams = 3;
2263 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2264 config.simulcast_layers[0].active = true;
2265 config.simulcast_layers[1].active = false;
2266 config.simulcast_layers[2].active = false;
2267 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432268 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132269 "VP8", /*max qp*/ 56, /*screencast*/ false,
2270 /*screenshare enabled*/ false);
2271 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2272
2273 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2274 // on lowest stream, limits for 270p should not be used
2275 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2276 EXPECT_FALSE(WaitForFrame(1000));
2277 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2278 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2279 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2280 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2281 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2282
2283 video_stream_encoder_->Stop();
2284}
2285
2286TEST_F(VideoStreamEncoderTest,
2287 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2288 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2289 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2290 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2291 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2292 fake_encoder_.SetResolutionBitrateLimits(
2293 {kEncoderLimits270p, kEncoderLimits360p});
2294 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2295
2296 // Two streams, highest stream active.
2297 VideoEncoderConfig config;
2298 const int kNumStreams = 2;
2299 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2300 config.simulcast_layers[0].active = false;
2301 config.simulcast_layers[1].active = true;
2302 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2303 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432304 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 14:36:132305 "VP8", /*max qp*/ 56, /*screencast*/ false,
2306 /*screenshare enabled*/ false);
2307 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2308
2309 // The encoder bitrate limits for 270p should be used.
2310 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2311 EXPECT_FALSE(WaitForFrame(1000));
2312 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2313 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2314 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2315 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2316 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2317
2318 // The max configured bitrate is less than the encoder limit for 360p.
2319 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2320 EXPECT_FALSE(WaitForFrame(1000));
2321 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2322 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2323 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2324 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2325
2326 video_stream_encoder_->Stop();
2327}
2328
mflodmancc3d4422017-08-03 15:27:512329TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 18:45:462330 EXPECT_TRUE(video_source_.has_sinks());
2331 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512332 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412333 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:462334 EXPECT_FALSE(video_source_.has_sinks());
2335 EXPECT_TRUE(new_video_source.has_sinks());
2336
mflodmancc3d4422017-08-03 15:27:512337 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462338}
2339
mflodmancc3d4422017-08-03 15:27:512340TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 18:45:462341 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512342 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 18:45:462343 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512344 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462345}
2346
Åsa Perssonc5a74ff2020-09-20 15:50:002347class ResolutionAlignmentTest
2348 : public VideoStreamEncoderTest,
2349 public ::testing::WithParamInterface<
2350 ::testing::tuple<int, std::vector<double>>> {
2351 public:
2352 ResolutionAlignmentTest()
2353 : requested_alignment_(::testing::get<0>(GetParam())),
2354 scale_factors_(::testing::get<1>(GetParam())) {}
2355
2356 protected:
2357 const int requested_alignment_;
2358 const std::vector<double> scale_factors_;
2359};
2360
2361INSTANTIATE_TEST_SUITE_P(
2362 AlignmentAndScaleFactors,
2363 ResolutionAlignmentTest,
2364 ::testing::Combine(
2365 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2366 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2367 std::vector<double>{-1.0, -1.0},
2368 std::vector<double>{-1.0, -1.0, -1.0},
2369 std::vector<double>{4.0, 2.0, 1.0},
2370 std::vector<double>{9999.0, -1.0, 1.0},
2371 std::vector<double>{3.99, 2.01, 1.0},
2372 std::vector<double>{4.9, 1.7, 1.25},
2373 std::vector<double>{10.0, 4.0, 3.0},
2374 std::vector<double>{1.75, 3.5},
2375 std::vector<double>{1.5, 2.5},
2376 std::vector<double>{1.3, 1.0})));
2377
2378TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2379 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 08:47:112380 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 15:50:002381 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2382 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2383
2384 // Fill config with the scaling factor by which to reduce encoding size.
2385 const int num_streams = scale_factors_.size();
2386 VideoEncoderConfig config;
2387 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2388 for (int i = 0; i < num_streams; ++i) {
2389 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2390 }
2391 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:432392 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 15:50:002393 "VP8", /*max qp*/ 56, /*screencast*/ false,
2394 /*screenshare enabled*/ false);
2395 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2396
Henrik Boström381d1092020-05-12 16:49:072397 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 15:50:002398 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2399 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2400 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2401 // Wait for all layers before triggering event.
2402 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 08:47:112403
2404 // On the 1st frame, we should have initialized the encoder and
2405 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 15:50:002406 int64_t timestamp_ms = kFrameIntervalMs;
2407 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2408 WaitForEncodedFrame(timestamp_ms);
2409 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 08:47:112410
2411 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2412 // (It's up the to the encoder to potentially drop the previous frame,
2413 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 15:50:002414 timestamp_ms += kFrameIntervalMs;
2415 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2416 WaitForEncodedFrame(timestamp_ms);
2417 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2418
2419 VideoCodec codec = fake_encoder_.video_codec();
2420 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2421 // Frame size should be a multiple of the requested alignment.
2422 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2423 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2424 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2425 // Aspect ratio should match.
2426 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2427 codec.height * codec.simulcastStream[i].width);
2428 }
Rasmus Brandt5cad55b2019-12-19 08:47:112429
2430 video_stream_encoder_->Stop();
2431}
2432
Jonathan Yubc771b72017-12-09 01:04:292433TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2434 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-14 06:25:222435 const int kWidth = 1280;
2436 const int kHeight = 720;
Jonathan Yubc771b72017-12-09 01:04:292437
2438 // We rely on the automatic resolution adaptation, but we handle framerate
2439 // adaptation manually by mocking the stats proxy.
2440 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-14 06:25:222441
Taylor Brandstetter49fcc102018-05-16 21:20:412442 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 16:49:072443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072444 DataRate::BitsPerSec(kTargetBitrateBps),
2445 DataRate::BitsPerSec(kTargetBitrateBps),
2446 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 21:20:412447 video_stream_encoder_->SetSource(&video_source_,
2448 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:392449 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:222450 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292451 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:222452 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2453
Jonathan Yubc771b72017-12-09 01:04:292454 // Adapt down as far as possible.
2455 rtc::VideoSinkWants last_wants;
2456 int64_t t = 1;
2457 int loop_count = 0;
2458 do {
2459 ++loop_count;
2460 last_wants = video_source_.sink_wants();
2461
2462 // Simulate the framerate we've been asked to adapt to.
2463 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2464 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2465 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2466 mock_stats.input_frame_rate = fps;
2467 stats_proxy_->SetMockStats(mock_stats);
2468
2469 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2470 sink_.WaitForEncodedFrame(t);
2471 t += frame_interval_ms;
2472
mflodmancc3d4422017-08-03 15:27:512473 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392474 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292475 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392476 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2477 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292478 } while (video_source_.sink_wants().max_pixel_count <
2479 last_wants.max_pixel_count ||
2480 video_source_.sink_wants().max_framerate_fps <
2481 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222482
Jonathan Yubc771b72017-12-09 01:04:292483 // Verify that we've adapted all the way down.
2484 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222485 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292486 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2487 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-14 06:25:222488 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:292489 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2490 *video_source_.last_sent_height());
2491 EXPECT_EQ(kMinBalancedFramerateFps,
2492 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222493
Jonathan Yubc771b72017-12-09 01:04:292494 // Adapt back up the same number of times we adapted down.
2495 for (int i = 0; i < loop_count - 1; ++i) {
2496 last_wants = video_source_.sink_wants();
2497
2498 // Simulate the framerate we've been asked to adapt to.
2499 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2500 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2501 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2502 mock_stats.input_frame_rate = fps;
2503 stats_proxy_->SetMockStats(mock_stats);
2504
2505 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2506 sink_.WaitForEncodedFrame(t);
2507 t += frame_interval_ms;
2508
Henrik Boström91aa7322020-04-28 10:24:332509 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392510 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292511 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392512 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2513 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292514 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2515 last_wants.max_pixel_count ||
2516 video_source_.sink_wants().max_framerate_fps >
2517 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222518 }
2519
Evan Shrubsole5cd7eb82020-05-25 12:08:392520 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-09 01:04:292521 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222522 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292523 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2524 EXPECT_EQ((loop_count - 1) * 2,
2525 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:222526
mflodmancc3d4422017-08-03 15:27:512527 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222528}
Rasmus Brandt5cad55b2019-12-19 08:47:112529
Evan Shrubsole2e2f6742020-05-14 08:41:152530TEST_F(VideoStreamEncoderTest,
2531 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2532 video_stream_encoder_->OnBitrateUpdated(
2533 DataRate::BitsPerSec(kTargetBitrateBps),
2534 DataRate::BitsPerSec(kTargetBitrateBps),
2535 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392536 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152537
2538 const int kFrameWidth = 1280;
2539 const int kFrameHeight = 720;
2540
2541 int64_t ntp_time = kFrameIntervalMs;
2542
2543 // Force an input frame rate to be available, or the adaptation call won't
2544 // know what framerate to adapt form.
2545 const int kInputFps = 30;
2546 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2547 stats.input_frame_rate = kInputFps;
2548 stats_proxy_->SetMockStats(stats);
2549
2550 video_source_.set_adaptation_enabled(true);
2551 video_stream_encoder_->SetSource(
2552 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 12:08:392553 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152554 video_source_.IncomingCapturedFrame(
2555 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2556 sink_.WaitForEncodedFrame(ntp_time);
2557 ntp_time += kFrameIntervalMs;
2558
2559 // Trigger CPU overuse.
2560 video_stream_encoder_->TriggerCpuOveruse();
2561 video_source_.IncomingCapturedFrame(
2562 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2563 sink_.WaitForEncodedFrame(ntp_time);
2564 ntp_time += kFrameIntervalMs;
2565
2566 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2567 EXPECT_EQ(std::numeric_limits<int>::max(),
2568 video_source_.sink_wants().max_pixel_count);
2569 // Some framerate constraint should be set.
2570 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2571 EXPECT_LT(restricted_fps, kInputFps);
2572 video_source_.IncomingCapturedFrame(
2573 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2574 sink_.WaitForEncodedFrame(ntp_time);
2575 ntp_time += 100;
2576
Henrik Boström2671dac2020-05-19 14:29:092577 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152578 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2579 // Give the encoder queue time to process the change in degradation preference
2580 // by waiting for an encoded frame.
2581 video_source_.IncomingCapturedFrame(
2582 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2583 sink_.WaitForEncodedFrame(ntp_time);
2584 ntp_time += kFrameIntervalMs;
2585
2586 video_stream_encoder_->TriggerQualityLow();
2587 video_source_.IncomingCapturedFrame(
2588 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2589 sink_.WaitForEncodedFrame(ntp_time);
2590 ntp_time += kFrameIntervalMs;
2591
2592 // Some resolution constraint should be set.
2593 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2594 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2595 kFrameWidth * kFrameHeight);
2596 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2597
2598 int pixel_count = video_source_.sink_wants().max_pixel_count;
2599 // Triggering a CPU underuse should not change the sink wants since it has
2600 // not been overused for resolution since we changed degradation preference.
2601 video_stream_encoder_->TriggerCpuUnderuse();
2602 video_source_.IncomingCapturedFrame(
2603 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2604 sink_.WaitForEncodedFrame(ntp_time);
2605 ntp_time += kFrameIntervalMs;
2606 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2607 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2608
Evan Shrubsole64469032020-06-11 08:45:292609 // Change the degradation preference back. CPU underuse should not adapt since
2610 // QP is most limited.
Henrik Boström2671dac2020-05-19 14:29:092611 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152612 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2613 video_source_.IncomingCapturedFrame(
2614 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2615 sink_.WaitForEncodedFrame(ntp_time);
2616 ntp_time += 100;
2617 // Resolution adaptations is gone after changing degradation preference.
2618 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2619 EXPECT_EQ(std::numeric_limits<int>::max(),
2620 video_source_.sink_wants().max_pixel_count);
2621 // The fps adaptation from above is now back.
2622 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2623
2624 // Trigger CPU underuse.
2625 video_stream_encoder_->TriggerCpuUnderuse();
2626 video_source_.IncomingCapturedFrame(
2627 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2628 sink_.WaitForEncodedFrame(ntp_time);
2629 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 08:45:292630 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2631
2632 // Trigger QP underuse, fps should return to normal.
2633 video_stream_encoder_->TriggerQualityHigh();
2634 video_source_.IncomingCapturedFrame(
2635 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2636 sink_.WaitForEncodedFrame(ntp_time);
2637 ntp_time += kFrameIntervalMs;
2638 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 08:41:152639
2640 video_stream_encoder_->Stop();
2641}
2642
mflodmancc3d4422017-08-03 15:27:512643TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 16:49:072644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072645 DataRate::BitsPerSec(kTargetBitrateBps),
2646 DataRate::BitsPerSec(kTargetBitrateBps),
2647 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392648 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 18:45:462649
sprangc5d62e22017-04-03 06:53:042650 const int kFrameWidth = 1280;
2651 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:042652
Åsa Persson8c1bf952018-09-13 08:42:192653 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 18:45:462654
kthelgason5e13d412016-12-01 11:59:512655 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042656 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522657 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042658 frame_timestamp += kFrameIntervalMs;
2659
perkj803d97f2016-11-01 18:45:462660 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512661 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462662 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042663 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522664 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042665 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 14:23:482666
asapersson0944a802017-04-07 07:57:582667 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-03 06:53:042668 // wanted resolution.
2669 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2670 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2671 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 08:42:192672 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042673
2674 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 18:45:462675 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 16:49:072676 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412677 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122678 // Give the encoder queue time to process the change in degradation preference
2679 // by waiting for an encoded frame.
2680 new_video_source.IncomingCapturedFrame(
2681 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2682 sink_.WaitForEncodedFrame(frame_timestamp);
2683 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042684 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 12:08:392685 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462686
sprangc5d62e22017-04-03 06:53:042687 // Force an input frame rate to be available, or the adaptation call won't
2688 // know what framerate to adapt form.
asapersson02465b82017-04-10 08:12:522689 const int kInputFps = 30;
sprangc5d62e22017-04-03 06:53:042690 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 08:12:522691 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-03 06:53:042692 stats_proxy_->SetMockStats(stats);
2693
mflodmancc3d4422017-08-03 15:27:512694 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462695 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042696 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522697 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042698 frame_timestamp += kFrameIntervalMs;
2699
2700 // Some framerate constraint should be set.
sprang84a37592017-02-10 15:04:272701 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-03 06:53:042702 EXPECT_EQ(std::numeric_limits<int>::max(),
2703 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522704 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-03 06:53:042705
asapersson02465b82017-04-10 08:12:522706 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 16:49:072707 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2708 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 10:42:122709 // Give the encoder queue time to process the change in degradation preference
2710 // by waiting for an encoded frame.
2711 new_video_source.IncomingCapturedFrame(
2712 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2713 sink_.WaitForEncodedFrame(frame_timestamp);
2714 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 12:08:392715 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-03 06:53:042716
mflodmancc3d4422017-08-03 15:27:512717 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042718 new_video_source.IncomingCapturedFrame(
2719 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522720 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042721 frame_timestamp += kFrameIntervalMs;
2722
2723 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 12:08:392724 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462725
2726 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072727 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412728 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 10:42:122729 // Give the encoder queue time to process the change in degradation preference
2730 // by waiting for an encoded frame.
2731 new_video_source.IncomingCapturedFrame(
2732 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2733 sink_.WaitForEncodedFrame(frame_timestamp);
2734 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042735 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2736 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 15:04:272737 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 08:42:192738 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042739
2740 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072741 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412742 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122743 // Give the encoder queue time to process the change in degradation preference
2744 // by waiting for an encoded frame.
2745 new_video_source.IncomingCapturedFrame(
2746 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2747 sink_.WaitForEncodedFrame(frame_timestamp);
2748 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042749 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2750 EXPECT_EQ(std::numeric_limits<int>::max(),
2751 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522752 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 18:45:462753
mflodmancc3d4422017-08-03 15:27:512754 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462755}
2756
mflodmancc3d4422017-08-03 15:27:512757TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072759 DataRate::BitsPerSec(kTargetBitrateBps),
2760 DataRate::BitsPerSec(kTargetBitrateBps),
2761 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 18:45:462762
asaperssonfab67072017-04-04 12:51:492763 const int kWidth = 1280;
2764 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492765 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522766 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 12:51:492767 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2768 EXPECT_FALSE(stats.bw_limited_resolution);
2769 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2770
2771 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512772 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492773 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522774 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 12:51:492775
2776 stats = stats_proxy_->GetStats();
2777 EXPECT_TRUE(stats.bw_limited_resolution);
2778 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2779
2780 // Trigger adapt up.
mflodmancc3d4422017-08-03 15:27:512781 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:492782 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522783 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 12:51:492784
2785 stats = stats_proxy_->GetStats();
2786 EXPECT_FALSE(stats.bw_limited_resolution);
2787 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2788 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2789
mflodmancc3d4422017-08-03 15:27:512790 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 12:51:492791}
2792
mflodmancc3d4422017-08-03 15:27:512793TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072794 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072795 DataRate::BitsPerSec(kTargetBitrateBps),
2796 DataRate::BitsPerSec(kTargetBitrateBps),
2797 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:492798
2799 const int kWidth = 1280;
2800 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492801 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522802 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 18:45:462803 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2804 EXPECT_FALSE(stats.cpu_limited_resolution);
2805 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2806
2807 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512808 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492809 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522810 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 18:45:462811
2812 stats = stats_proxy_->GetStats();
2813 EXPECT_TRUE(stats.cpu_limited_resolution);
2814 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2815
2816 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332817 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492818 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522819 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 18:45:462820
2821 stats = stats_proxy_->GetStats();
2822 EXPECT_FALSE(stats.cpu_limited_resolution);
2823 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 12:51:492824 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:462825
mflodmancc3d4422017-08-03 15:27:512826 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462827}
2828
mflodmancc3d4422017-08-03 15:27:512829TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072831 DataRate::BitsPerSec(kTargetBitrateBps),
2832 DataRate::BitsPerSec(kTargetBitrateBps),
2833 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112834
asaperssonfab67072017-04-04 12:51:492835 const int kWidth = 1280;
2836 const int kHeight = 720;
2837 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522838 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112839 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182840 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112841 EXPECT_FALSE(stats.cpu_limited_resolution);
2842 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2843
asaperssonfab67072017-04-04 12:51:492844 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512845 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492846 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522847 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112848 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182849 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112850 EXPECT_TRUE(stats.cpu_limited_resolution);
2851 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2852
2853 // Set new source with adaptation still enabled.
2854 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512855 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412856 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112857
asaperssonfab67072017-04-04 12:51:492858 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522859 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112860 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182861 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112862 EXPECT_TRUE(stats.cpu_limited_resolution);
2863 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2864
2865 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412866 video_stream_encoder_->SetSource(&new_video_source,
2867 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 09:44:112868
asaperssonfab67072017-04-04 12:51:492869 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522870 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112871 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182872 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112873 EXPECT_FALSE(stats.cpu_limited_resolution);
2874 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2875
2876 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 15:27:512877 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412878 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112879
asaperssonfab67072017-04-04 12:51:492880 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522881 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112882 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182883 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112884 EXPECT_TRUE(stats.cpu_limited_resolution);
2885 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2886
asaperssonfab67072017-04-04 12:51:492887 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332888 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492889 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522890 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 09:44:112891 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182892 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112893 EXPECT_FALSE(stats.cpu_limited_resolution);
2894 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:522895 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112896
mflodmancc3d4422017-08-03 15:27:512897 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112898}
2899
mflodmancc3d4422017-08-03 15:27:512900TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072902 DataRate::BitsPerSec(kTargetBitrateBps),
2903 DataRate::BitsPerSec(kTargetBitrateBps),
2904 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112905
asaperssonfab67072017-04-04 12:51:492906 const int kWidth = 1280;
2907 const int kHeight = 720;
2908 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522909 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112910 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112911 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022912 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492913 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112914
2915 // Set new source with adaptation still enabled.
2916 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 21:20:412917 video_stream_encoder_->SetSource(&new_video_source,
2918 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112919
asaperssonfab67072017-04-04 12:51:492920 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522921 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112922 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112923 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022924 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492925 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112926
asaperssonfab67072017-04-04 12:51:492927 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512928 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492929 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522930 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112931 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112932 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022933 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492934 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112935
asaperssonfab67072017-04-04 12:51:492936 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412937 video_stream_encoder_->SetSource(&new_video_source,
2938 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112939
asaperssonfab67072017-04-04 12:51:492940 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522941 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112942 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112943 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022944 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492945 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112946
asapersson02465b82017-04-10 08:12:522947 // Disable resolution scaling.
mflodmancc3d4422017-08-03 15:27:512948 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412949 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:112950
asaperssonfab67072017-04-04 12:51:492951 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522952 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112953 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112954 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022955 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492956 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2957 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 09:44:112958
mflodmancc3d4422017-08-03 15:27:512959 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112960}
2961
mflodmancc3d4422017-08-03 15:27:512962TEST_F(VideoStreamEncoderTest,
2963 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 16:49:072964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072965 DataRate::BitsPerSec(kTargetBitrateBps),
2966 DataRate::BitsPerSec(kTargetBitrateBps),
2967 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 12:29:122968
2969 const int kWidth = 1280;
2970 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 08:42:192971 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 12:29:122972 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 08:42:192973 video_source_.IncomingCapturedFrame(
2974 CreateFrame(timestamp_ms, kWidth, kHeight));
2975 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122976 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2977 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2978 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2979
2980 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512981 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:192982 timestamp_ms += kFrameIntervalMs;
2983 video_source_.IncomingCapturedFrame(
2984 CreateFrame(timestamp_ms, kWidth, kHeight));
2985 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122986 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2987 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2988 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2989
2990 // Trigger overuse.
mflodmancc3d4422017-08-03 15:27:512991 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:192992 timestamp_ms += kFrameIntervalMs;
2993 video_source_.IncomingCapturedFrame(
2994 CreateFrame(timestamp_ms, kWidth, kHeight));
2995 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122996 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2997 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2998 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2999
Niels Möller4db138e2018-04-19 07:04:133000 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 12:29:123001 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:133002
3003 VideoEncoderConfig video_encoder_config;
3004 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3005 // Make format different, to force recreation of encoder.
3006 video_encoder_config.video_format.parameters["foo"] = "foo";
3007 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473008 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 08:42:193009 timestamp_ms += kFrameIntervalMs;
3010 video_source_.IncomingCapturedFrame(
3011 CreateFrame(timestamp_ms, kWidth, kHeight));
3012 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:123013 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3014 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3015 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3016
mflodmancc3d4422017-08-03 15:27:513017 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 12:29:123018}
3019
mflodmancc3d4422017-08-03 15:27:513020TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 17:39:323021 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 16:49:073022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 17:39:323023 DataRate::BitsPerSec(kTargetBitrateBps),
3024 DataRate::BitsPerSec(kTargetBitrateBps),
3025 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3026
3027 const int kWidth = 1280;
3028 const int kHeight = 720;
3029 int sequence = 1;
3030
3031 // Enable BALANCED preference, no initial limitation.
3032 test::FrameForwarder source;
3033 video_stream_encoder_->SetSource(&source,
3034 webrtc::DegradationPreference::BALANCED);
3035 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3036 WaitForEncodedFrame(sequence++);
3037 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3038 EXPECT_FALSE(stats.cpu_limited_resolution);
3039 EXPECT_FALSE(stats.cpu_limited_framerate);
3040 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3041
3042 // Trigger CPU overuse, should now adapt down.
3043 video_stream_encoder_->TriggerCpuOveruse();
3044 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3045 WaitForEncodedFrame(sequence++);
3046 stats = stats_proxy_->GetStats();
3047 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3048
3049 // Set new degradation preference should clear restrictions since we changed
3050 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 14:54:213051 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 17:39:323052 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3053 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3054 WaitForEncodedFrame(sequence++);
3055 stats = stats_proxy_->GetStats();
3056 EXPECT_FALSE(stats.cpu_limited_resolution);
3057 EXPECT_FALSE(stats.cpu_limited_framerate);
3058 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3059
3060 // Force an input frame rate to be available, or the adaptation call won't
3061 // know what framerate to adapt from.
3062 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3063 mock_stats.input_frame_rate = 30;
3064 stats_proxy_->SetMockStats(mock_stats);
3065 video_stream_encoder_->TriggerCpuOveruse();
3066 stats_proxy_->ResetMockStats();
3067 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3068 WaitForEncodedFrame(sequence++);
3069
3070 // We have now adapted once.
3071 stats = stats_proxy_->GetStats();
3072 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3073
3074 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 14:54:213075 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3076 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 17:39:323077 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3078 WaitForEncodedFrame(sequence++);
3079 stats = stats_proxy_->GetStats();
3080 EXPECT_FALSE(stats.cpu_limited_resolution);
3081 EXPECT_FALSE(stats.cpu_limited_framerate);
3082 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3083
3084 video_stream_encoder_->Stop();
3085}
3086
3087TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513088 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 16:49:073089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073090 DataRate::BitsPerSec(kTargetBitrateBps),
3091 DataRate::BitsPerSec(kTargetBitrateBps),
3092 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 18:45:463093
asapersson0944a802017-04-07 07:57:583094 const int kWidth = 1280;
3095 const int kHeight = 720;
sprang84a37592017-02-10 15:04:273096 int sequence = 1;
perkj803d97f2016-11-01 18:45:463097
asaperssonfab67072017-04-04 12:51:493098 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523099 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463100 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 15:04:273101 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023102 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 15:04:273103 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3104
asapersson02465b82017-04-10 08:12:523105 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 15:27:513106 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:493107 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523108 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 15:04:273109 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 18:45:463110 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023111 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463112 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3113
3114 // Set new source with adaptation still enabled.
3115 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513116 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413117 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:463118
3119 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493120 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523121 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463122 stats = stats_proxy_->GetStats();
3123 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183124 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463125 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3126
sprangc5d62e22017-04-03 06:53:043127 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 15:27:513128 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413129 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 18:45:463130 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493131 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523132 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463133 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-03 06:53:043134 // Not adapted at first.
perkj803d97f2016-11-01 18:45:463135 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183136 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463137 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3138
sprangc5d62e22017-04-03 06:53:043139 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-16 06:40:183140 // know what framerate to adapt from.
sprangc5d62e22017-04-03 06:53:043141 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3142 mock_stats.input_frame_rate = 30;
3143 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513144 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043145 stats_proxy_->ResetMockStats();
3146
3147 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493148 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523149 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043150
3151 // Framerate now adapted.
3152 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:183153 EXPECT_FALSE(stats.cpu_limited_resolution);
3154 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043155 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3156
3157 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 21:20:413158 video_stream_encoder_->SetSource(&new_video_source,
3159 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-03 06:53:043160 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493161 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523162 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043163
3164 stats = stats_proxy_->GetStats();
3165 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023166 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043167 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3168
3169 // Try to trigger overuse. Should not succeed.
3170 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513171 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043172 stats_proxy_->ResetMockStats();
3173
3174 stats = stats_proxy_->GetStats();
3175 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023176 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043177 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3178
3179 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 15:27:513180 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413181 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 12:51:493182 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523183 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463184 stats = stats_proxy_->GetStats();
3185 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023186 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043187 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 18:45:463188
3189 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333190 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:493191 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523192 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463193 stats = stats_proxy_->GetStats();
3194 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023195 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043196 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3197
3198 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 15:27:513199 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413200 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:043201 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493202 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523203 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043204 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 07:01:023205 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-03 06:53:043206 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023207 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043208 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3209
3210 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333211 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:043212 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493213 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523214 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043215 stats = stats_proxy_->GetStats();
3216 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023217 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043218 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:523219 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:463220
mflodmancc3d4422017-08-03 15:27:513221 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:463222}
3223
mflodmancc3d4422017-08-03 15:27:513224TEST_F(VideoStreamEncoderTest,
3225 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 12:51:493226 const int kWidth = 1280;
3227 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073229 DataRate::BitsPerSec(kTargetBitrateBps),
3230 DataRate::BitsPerSec(kTargetBitrateBps),
3231 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:113232
asaperssonfab67072017-04-04 12:51:493233 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 12:08:393234 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 09:44:113235
asaperssonfab67072017-04-04 12:51:493236 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523237 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:113238
asaperssonfab67072017-04-04 12:51:493239 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513240 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 11:59:513241
asaperssonfab67072017-04-04 12:51:493242 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523243 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 11:59:513244
kthelgason876222f2016-11-29 09:44:113245 // Expect a scale down.
3246 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 12:51:493247 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 09:44:113248
asapersson02465b82017-04-10 08:12:523249 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 09:44:113250 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513251 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413252 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:113253
asaperssonfab67072017-04-04 12:51:493254 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513255 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:493256 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523257 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:113258
asaperssonfab67072017-04-04 12:51:493259 // Expect no scaling.
sprangc5d62e22017-04-03 06:53:043260 EXPECT_EQ(std::numeric_limits<int>::max(),
3261 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113262
asaperssonfab67072017-04-04 12:51:493263 // Trigger scale up.
mflodmancc3d4422017-08-03 15:27:513264 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:493265 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523266 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:113267
asapersson02465b82017-04-10 08:12:523268 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-03 06:53:043269 EXPECT_EQ(std::numeric_limits<int>::max(),
3270 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113271
mflodmancc3d4422017-08-03 15:27:513272 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:113273}
3274
mflodmancc3d4422017-08-03 15:27:513275TEST_F(VideoStreamEncoderTest,
3276 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523277 const int kWidth = 1280;
3278 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073279 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073280 DataRate::BitsPerSec(kTargetBitrateBps),
3281 DataRate::BitsPerSec(kTargetBitrateBps),
3282 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:523283
Taylor Brandstetter49fcc102018-05-16 21:20:413284 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523285 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513286 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413287 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523288
3289 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523290 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393291 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523292 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3293 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3294
3295 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513296 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393297 EXPECT_THAT(source.sink_wants(),
3298 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523299 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3300 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3301 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3302
3303 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 15:27:513304 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 08:12:523305 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3306 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3307 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3308
mflodmancc3d4422017-08-03 15:27:513309 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523310}
3311
mflodmancc3d4422017-08-03 15:27:513312TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223313 const int kWidth = 1280;
3314 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073316 DataRate::BitsPerSec(kTargetBitrateBps),
3317 DataRate::BitsPerSec(kTargetBitrateBps),
3318 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223319
Taylor Brandstetter49fcc102018-05-16 21:20:413320 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223321 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413322 video_stream_encoder_->SetSource(&source,
3323 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223324 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3325 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393326 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223327
3328 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513329 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:393330 EXPECT_THAT(source.sink_wants(),
3331 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223332 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3333 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3334 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3335
3336 // Trigger adapt down for same input resolution, expect no change.
3337 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3338 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:513339 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223340 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3341 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3342 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3343
3344 // Trigger adapt down for larger input resolution, expect no change.
3345 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3346 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:513347 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223348 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3349 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3350 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3351
mflodmancc3d4422017-08-03 15:27:513352 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223353}
3354
mflodmancc3d4422017-08-03 15:27:513355TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 20:55:383356 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3357 const int kWidth = 640;
3358 const int kHeight = 360;
3359 const int64_t kFrameIntervalMs = 150;
3360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
3361 DataRate::BitsPerSec(kTargetBitrateBps),
3362 DataRate::BitsPerSec(kTargetBitrateBps),
3363 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3364
3365 // Enable BALANCED preference, no initial limitation.
3366 AdaptingFrameForwarder source(&time_controller_);
3367 source.set_adaptation_enabled(true);
3368 video_stream_encoder_->SetSource(&source,
3369 webrtc::DegradationPreference::BALANCED);
3370
3371 int64_t timestamp_ms = kFrameIntervalMs;
3372 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3373 sink_.WaitForEncodedFrame(kWidth, kHeight);
3374 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3375 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3376 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3377 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3378
3379 // Trigger adapt down, expect reduced fps (640x360@15fps).
3380 video_stream_encoder_->TriggerQualityLow();
3381 timestamp_ms += kFrameIntervalMs;
3382 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3383 sink_.WaitForEncodedFrame(timestamp_ms);
3384 EXPECT_THAT(source.sink_wants(),
3385 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3386 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3387 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3388 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3389
3390 // Source requests 270p, expect reduced resolution (480x270@15fps).
3391 source.OnOutputFormatRequest(480, 270);
3392 timestamp_ms += kFrameIntervalMs;
3393 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3394 WaitForEncodedFrame(480, 270);
3395 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3396
3397 // Trigger adapt down, expect reduced fps (480x270@10fps).
3398 video_stream_encoder_->TriggerQualityLow();
3399 timestamp_ms += kFrameIntervalMs;
3400 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3401 sink_.WaitForEncodedFrame(timestamp_ms);
3402 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3404 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3405 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3406
3407 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3408 source.OnOutputFormatRequest(320, 180);
3409 timestamp_ms += kFrameIntervalMs;
3410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3411 WaitForEncodedFrame(320, 180);
3412 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3413
3414 // Trigger adapt down, expect reduced fps (320x180@7fps).
3415 video_stream_encoder_->TriggerQualityLow();
3416 timestamp_ms += kFrameIntervalMs;
3417 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3418 sink_.WaitForEncodedFrame(timestamp_ms);
3419 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3420 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3421 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3422 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3423
3424 // Source requests VGA, expect increased resolution (640x360@7fps).
3425 source.OnOutputFormatRequest(640, 360);
3426 timestamp_ms += kFrameIntervalMs;
3427 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3428 WaitForEncodedFrame(timestamp_ms);
3429 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3430
3431 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3432 video_stream_encoder_->TriggerQualityHigh();
3433 timestamp_ms += kFrameIntervalMs;
3434 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3435 WaitForEncodedFrame(timestamp_ms);
3436 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3437 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3439 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3440
3441 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3442 video_stream_encoder_->TriggerQualityHigh();
3443 timestamp_ms += kFrameIntervalMs;
3444 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3445 WaitForEncodedFrame(timestamp_ms);
3446 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3447 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3448 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3449 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3450
3451 // Trigger adapt up, expect increased fps (640x360@maxfps).
3452 video_stream_encoder_->TriggerQualityHigh();
3453 timestamp_ms += kFrameIntervalMs;
3454 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3455 WaitForEncodedFrame(timestamp_ms);
3456 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3457 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3459 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 video_stream_encoder_->Stop();
3462}
3463
3464TEST_F(VideoStreamEncoderTest,
3465 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3466 const int kWidth = 1280;
3467 const int kHeight = 720;
3468 const int64_t kFrameIntervalMs = 150;
3469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
3470 DataRate::BitsPerSec(kTargetBitrateBps),
3471 DataRate::BitsPerSec(kTargetBitrateBps),
3472 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3473
3474 // Enable BALANCED preference, no initial limitation.
3475 AdaptingFrameForwarder source(&time_controller_);
3476 source.set_adaptation_enabled(true);
3477 video_stream_encoder_->SetSource(&source,
3478 webrtc::DegradationPreference::BALANCED);
3479
3480 int64_t timestamp_ms = kFrameIntervalMs;
3481 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3482 sink_.WaitForEncodedFrame(kWidth, kHeight);
3483 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3484 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3486 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3487
3488 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3489 video_stream_encoder_->TriggerQualityLow();
3490 timestamp_ms += kFrameIntervalMs;
3491 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3492 sink_.WaitForEncodedFrame(timestamp_ms);
3493 EXPECT_THAT(source.sink_wants(),
3494 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3496 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3497 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3498
3499 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3500 video_stream_encoder_->TriggerQualityLow();
3501 timestamp_ms += kFrameIntervalMs;
3502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3503 sink_.WaitForEncodedFrame(timestamp_ms);
3504 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3505 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3506 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3507 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3508
3509 // Trigger adapt down, expect reduced fps (640x360@15fps).
3510 video_stream_encoder_->TriggerQualityLow();
3511 timestamp_ms += kFrameIntervalMs;
3512 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3513 WaitForEncodedFrame(timestamp_ms);
3514 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3517 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3518
3519 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3520 source.OnOutputFormatRequest(320, 180);
3521 timestamp_ms += kFrameIntervalMs;
3522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3523 WaitForEncodedFrame(320, 180);
3524 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3525 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3526
3527 // Trigger adapt down, expect reduced fps (320x180@7fps).
3528 video_stream_encoder_->TriggerCpuOveruse();
3529 timestamp_ms += kFrameIntervalMs;
3530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3531 WaitForEncodedFrame(timestamp_ms);
3532 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3533 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3535 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3536 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3537 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3538 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3539
3540 // Source requests HD, expect increased resolution (640x360@7fps).
3541 source.OnOutputFormatRequest(1280, 720);
3542 timestamp_ms += kFrameIntervalMs;
3543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3544 WaitForEncodedFrame(timestamp_ms);
3545 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3546 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3547
3548 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3549 video_stream_encoder_->TriggerCpuUnderuse();
3550 timestamp_ms += kFrameIntervalMs;
3551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3552 WaitForEncodedFrame(timestamp_ms);
3553 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3556 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3557 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3558 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3559 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3560
3561 // Trigger adapt up, expect increased fps (640x360@maxfps).
3562 video_stream_encoder_->TriggerQualityHigh();
3563 video_stream_encoder_->TriggerCpuUnderuse();
3564 timestamp_ms += kFrameIntervalMs;
3565 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3566 WaitForEncodedFrame(timestamp_ms);
3567 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3569 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3570 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3571 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3572 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3573 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3574
3575 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3576 video_stream_encoder_->TriggerQualityHigh();
3577 video_stream_encoder_->TriggerCpuUnderuse();
3578 timestamp_ms += kFrameIntervalMs;
3579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3580 WaitForEncodedFrame(timestamp_ms);
3581 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3583 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3584 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3585 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3586 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3588
3589 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3590 video_stream_encoder_->TriggerQualityHigh();
3591 video_stream_encoder_->TriggerCpuUnderuse();
3592 timestamp_ms += kFrameIntervalMs;
3593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3594 WaitForEncodedFrame(timestamp_ms);
3595 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3597 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3598 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3599 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3600 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3601 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3602
3603 video_stream_encoder_->Stop();
3604}
3605
3606TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513607 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523608 const int kWidth = 1280;
3609 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073611 DataRate::BitsPerSec(kTargetBitrateBps),
3612 DataRate::BitsPerSec(kTargetBitrateBps),
3613 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:523614
Taylor Brandstetter49fcc102018-05-16 21:20:413615 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523616 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513617 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413618 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523619
3620 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523621 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393622 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523623 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3624 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3625
3626 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333627 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393628 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523629 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3630 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3631
mflodmancc3d4422017-08-03 15:27:513632 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523633}
3634
mflodmancc3d4422017-08-03 15:27:513635TEST_F(VideoStreamEncoderTest,
3636 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 08:12:523637 const int kWidth = 1280;
3638 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073640 DataRate::BitsPerSec(kTargetBitrateBps),
3641 DataRate::BitsPerSec(kTargetBitrateBps),
3642 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:523643
Taylor Brandstetter49fcc102018-05-16 21:20:413644 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523645 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513646 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413647 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 08:12:523648
3649 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523650 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393651 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183652 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3654
3655 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333656 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393657 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183658 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523659 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3660
mflodmancc3d4422017-08-03 15:27:513661 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523662}
3663
mflodmancc3d4422017-08-03 15:27:513664TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223665 const int kWidth = 1280;
3666 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073668 DataRate::BitsPerSec(kTargetBitrateBps),
3669 DataRate::BitsPerSec(kTargetBitrateBps),
3670 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223671
Taylor Brandstetter49fcc102018-05-16 21:20:413672 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223673 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413674 video_stream_encoder_->SetSource(&source,
3675 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223676
3677 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3678 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393679 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223680 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3681 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3682 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3683
3684 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513685 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393686 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223687 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3688 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3689 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3690
mflodmancc3d4422017-08-03 15:27:513691 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223692}
3693
mflodmancc3d4422017-08-03 15:27:513694TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-16 06:40:183695 const int kWidth = 1280;
3696 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073698 DataRate::BitsPerSec(kTargetBitrateBps),
3699 DataRate::BitsPerSec(kTargetBitrateBps),
3700 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:183701
Taylor Brandstetter49fcc102018-05-16 21:20:413702 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-16 06:40:183703 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413704 video_stream_encoder_->SetSource(&source,
3705 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-16 06:40:183706
3707 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3708 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393709 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183710 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3711 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3712 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3713
3714 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513715 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393716 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183717 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3719 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3720
mflodmancc3d4422017-08-03 15:27:513721 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183722}
3723
mflodmancc3d4422017-08-03 15:27:513724TEST_F(VideoStreamEncoderTest,
3725 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523726 const int kWidth = 1280;
3727 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073729 DataRate::BitsPerSec(kTargetBitrateBps),
3730 DataRate::BitsPerSec(kTargetBitrateBps),
3731 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:523732
Taylor Brandstetter49fcc102018-05-16 21:20:413733 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233734 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 08:12:523735 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513736 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413737 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523738
3739 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523740 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393741 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523742 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3743 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3744
3745 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513746 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 08:12:523747 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523748 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393749 EXPECT_THAT(source.sink_wants(),
3750 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523751 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3752 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3753
3754 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513755 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393756 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523757 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3758 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3759 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3760
mflodmancc3d4422017-08-03 15:27:513761 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523762}
3763
mflodmancc3d4422017-08-03 15:27:513764TEST_F(VideoStreamEncoderTest,
3765 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-16 06:40:183766 const int kWidth = 1280;
3767 const int kHeight = 720;
3768 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 16:49:073769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073770 DataRate::BitsPerSec(kTargetBitrateBps),
3771 DataRate::BitsPerSec(kTargetBitrateBps),
3772 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:183773
3774 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3775 stats.input_frame_rate = kInputFps;
3776 stats_proxy_->SetMockStats(stats);
3777
Taylor Brandstetter49fcc102018-05-16 21:20:413778 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-16 06:40:183779 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3780 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393781 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183782
3783 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513784 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-16 06:40:183785 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3786 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393787 EXPECT_THAT(video_source_.sink_wants(),
3788 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-16 06:40:183789
Taylor Brandstetter49fcc102018-05-16 21:20:413790 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-16 06:40:183791 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 14:29:093792 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:413793 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:123794 // Give the encoder queue time to process the change in degradation preference
3795 // by waiting for an encoded frame.
3796 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3797 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 12:08:393798 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183799
3800 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 15:27:513801 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 10:42:123802 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3803 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 12:08:393804 EXPECT_THAT(new_video_source.sink_wants(),
3805 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-16 06:40:183806
3807 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513808 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393809 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183810
mflodmancc3d4422017-08-03 15:27:513811 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183812}
3813
mflodmancc3d4422017-08-03 15:27:513814TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 08:47:313815 const int kWidth = 1280;
3816 const int kHeight = 720;
3817 const size_t kNumFrames = 10;
3818
Henrik Boström381d1092020-05-12 16:49:073819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073820 DataRate::BitsPerSec(kTargetBitrateBps),
3821 DataRate::BitsPerSec(kTargetBitrateBps),
3822 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 11:59:513823
asaperssond0de2952017-04-21 08:47:313824 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 15:58:543825 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 08:47:313826 video_source_.set_adaptation_enabled(true);
3827
3828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3829 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3830
3831 int downscales = 0;
3832 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 08:42:193833 video_source_.IncomingCapturedFrame(
3834 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3835 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 08:47:313836
asaperssonfab67072017-04-04 12:51:493837 // Trigger scale down.
asaperssond0de2952017-04-21 08:47:313838 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 15:27:513839 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-03 06:53:043840 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 08:47:313841
3842 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3843 ++downscales;
3844
3845 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3846 EXPECT_EQ(downscales,
3847 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3848 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 11:59:513849 }
mflodmancc3d4422017-08-03 15:27:513850 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313851}
3852
mflodmancc3d4422017-08-03 15:27:513853TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:313854 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3855 const int kWidth = 1280;
3856 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073858 DataRate::BitsPerSec(kTargetBitrateBps),
3859 DataRate::BitsPerSec(kTargetBitrateBps),
3860 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:313861
Taylor Brandstetter49fcc102018-05-16 21:20:413862 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233863 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:313864 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513865 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413866 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:313867
Åsa Persson8c1bf952018-09-13 08:42:193868 int64_t timestamp_ms = kFrameIntervalMs;
3869 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523870 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393871 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313872 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3873 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3874
3875 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513876 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193877 timestamp_ms += kFrameIntervalMs;
3878 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3879 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393880 EXPECT_THAT(source.sink_wants(),
3881 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313882 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3883 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3884
3885 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333886 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193887 timestamp_ms += kFrameIntervalMs;
3888 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523889 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393890 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313891 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3892 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3893
3894 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513895 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193896 timestamp_ms += kFrameIntervalMs;
3897 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3898 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393899 EXPECT_THAT(source.sink_wants(),
3900 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313901 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3902 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3903
3904 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333905 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193906 timestamp_ms += kFrameIntervalMs;
3907 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-16 06:40:183908 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393909 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313910 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3911 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3912
mflodmancc3d4422017-08-03 15:27:513913 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313914}
3915
mflodmancc3d4422017-08-03 15:27:513916TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-14 06:25:223917 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3918 const int kWidth = 1280;
3919 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073921 DataRate::BitsPerSec(kTargetBitrateBps),
3922 DataRate::BitsPerSec(kTargetBitrateBps),
3923 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223924
Taylor Brandstetter49fcc102018-05-16 21:20:413925 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233926 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:223927 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:413928 video_stream_encoder_->SetSource(&source,
3929 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223930
Åsa Persson8c1bf952018-09-13 08:42:193931 int64_t timestamp_ms = kFrameIntervalMs;
3932 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223933 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393934 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223935 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3936 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3937
3938 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513939 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193940 timestamp_ms += kFrameIntervalMs;
3941 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3942 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393943 EXPECT_THAT(source.sink_wants(),
3944 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223945 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3946 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3947
3948 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513949 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193950 timestamp_ms += kFrameIntervalMs;
3951 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223952 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393953 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223954 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3955 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3956
3957 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513958 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193959 timestamp_ms += kFrameIntervalMs;
3960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3961 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393962 EXPECT_THAT(source.sink_wants(),
3963 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223964 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3965 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3966
3967 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513968 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193969 timestamp_ms += kFrameIntervalMs;
3970 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223971 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393972 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223973 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3974 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3975
mflodmancc3d4422017-08-03 15:27:513976 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223977}
3978
Sergey Silkin41c650b2019-10-14 11:12:193979TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3980 fake_encoder_.SetResolutionBitrateLimits(
3981 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3982
Henrik Boström381d1092020-05-12 16:49:073983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073984 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3985 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3986 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3987 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:193988
3989 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233990 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:193991 source.set_adaptation_enabled(true);
3992 video_stream_encoder_->SetSource(
3993 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3994
3995 // Insert 720p frame.
3996 int64_t timestamp_ms = kFrameIntervalMs;
3997 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3998 WaitForEncodedFrame(1280, 720);
3999
4000 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 16:49:074001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074002 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4003 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4004 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4005 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194006 video_stream_encoder_->TriggerQualityLow();
4007
4008 // Insert 720p frame. It should be downscaled and encoded.
4009 timestamp_ms += kFrameIntervalMs;
4010 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4011 WaitForEncodedFrame(960, 540);
4012
4013 // Trigger adapt up. Higher resolution should not be requested duo to lack
4014 // of bitrate.
4015 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:394016 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 11:12:194017
4018 // Increase bitrate.
Henrik Boström381d1092020-05-12 16:49:074019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074020 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4021 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4022 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4023 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194024
4025 // Trigger adapt up. Higher resolution should be requested.
4026 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:394027 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 11:12:194028
4029 video_stream_encoder_->Stop();
4030}
4031
4032TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4033 fake_encoder_.SetResolutionBitrateLimits(
4034 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4035
4036 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 16:49:074037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074038 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4039 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4040 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4041 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:194042
4043 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234044 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:194045 source.set_adaptation_enabled(true);
4046 video_stream_encoder_->SetSource(
4047 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4048
4049 // Insert 720p frame. It should be dropped and lower resolution should be
4050 // requested.
4051 int64_t timestamp_ms = kFrameIntervalMs;
4052 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4053 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:094054 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 11:12:194055
4056 // Insert 720p frame. It should be downscaled and encoded.
4057 timestamp_ms += kFrameIntervalMs;
4058 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4059 WaitForEncodedFrame(960, 540);
4060
4061 video_stream_encoder_->Stop();
4062}
4063
Åsa Perssonb67c44c2019-09-24 13:25:324064class BalancedDegradationTest : public VideoStreamEncoderTest {
4065 protected:
4066 void SetupTest() {
4067 // Reset encoder for field trials to take effect.
4068 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 13:13:044069 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 13:25:324070
4071 // Enable BALANCED preference.
4072 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 13:13:044073 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4074 }
4075
4076 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 16:49:074077 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074078 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
4079 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 13:25:324080 }
4081
Åsa Persson45b176f2019-09-30 09:19:054082 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 13:25:324083 timestamp_ms_ += kFrameIntervalMs;
4084 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 09:19:054085 }
4086
4087 void InsertFrameAndWaitForEncoded() {
4088 InsertFrame();
Åsa Perssonb67c44c2019-09-24 13:25:324089 sink_.WaitForEncodedFrame(timestamp_ms_);
4090 }
4091
4092 const int kWidth = 640; // pixels:640x360=230400
4093 const int kHeight = 360;
4094 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4095 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:234096 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 13:25:324097};
4098
Evan Shrubsolea1c77f62020-08-10 09:01:064099TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 13:25:324100 test::ScopedFieldTrials field_trials(
4101 "WebRTC-Video-BalancedDegradationSettings/"
4102 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4103 SetupTest();
4104
4105 // Force input frame rate.
4106 const int kInputFps = 24;
4107 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4108 stats.input_frame_rate = kInputFps;
4109 stats_proxy_->SetMockStats(stats);
4110
Åsa Persson45b176f2019-09-30 09:19:054111 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394112 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324113
Evan Shrubsolea1c77f62020-08-10 09:01:064114 // Trigger adapt down, expect scaled down framerate and resolution,
4115 // since Fps diff (input-requested:0) < threshold.
4116 video_stream_encoder_->TriggerQualityLow();
4117 EXPECT_THAT(source_.sink_wants(),
4118 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 13:25:324119
4120 video_stream_encoder_->Stop();
4121}
4122
Evan Shrubsolea1c77f62020-08-10 09:01:064123TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 13:25:324124 test::ScopedFieldTrials field_trials(
4125 "WebRTC-Video-BalancedDegradationSettings/"
4126 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4127 SetupTest();
4128
4129 // Force input frame rate.
4130 const int kInputFps = 25;
4131 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4132 stats.input_frame_rate = kInputFps;
4133 stats_proxy_->SetMockStats(stats);
4134
Åsa Persson45b176f2019-09-30 09:19:054135 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394136 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324137
Evan Shrubsolea1c77f62020-08-10 09:01:064138 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4139 // Fps diff (input-requested:1) == threshold.
4140 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:394141 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 13:25:324142
4143 video_stream_encoder_->Stop();
4144}
4145
4146TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4147 test::ScopedFieldTrials field_trials(
4148 "WebRTC-Video-BalancedDegradationSettings/"
4149 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4150 SetupTest();
4151
4152 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4153
Åsa Persson45b176f2019-09-30 09:19:054154 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394155 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:324156
4157 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4158 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:394159 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 13:25:324160
4161 video_stream_encoder_->Stop();
4162}
4163
Åsa Perssonccfb3402019-09-25 13:13:044164TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324165 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 15:26:394166 "WebRTC-Video-BalancedDegradationSettings/"
4167 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 13:13:044168 SetupTest();
Åsa Persson1b247f12019-08-14 15:26:394169
Åsa Persson1b247f12019-08-14 15:26:394170 const int kMinBitrateBps = 425000;
4171 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 13:13:044172 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 15:26:394173
Åsa Persson45b176f2019-09-30 09:19:054174 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394175 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 15:26:394176 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4177
4178 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4179 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054180 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394181 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 15:26:394182 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4183
4184 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4185 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054186 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544187 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:394188 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4189
Åsa Persson30ab0152019-08-27 10:22:334190 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4191 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054192 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544193 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 13:13:044194 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 10:22:334195 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4196
4197 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 15:26:394198 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054199 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334200 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 15:26:394201
Åsa Persson30ab0152019-08-27 10:22:334202 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044203 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 15:26:394204 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054205 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:044206 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 10:22:334207 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4208
4209 video_stream_encoder_->Stop();
4210}
4211
Åsa Perssonccfb3402019-09-25 13:13:044212TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 09:19:054213 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4214 test::ScopedFieldTrials field_trials(
4215 "WebRTC-Video-BalancedDegradationSettings/"
4216 "pixels:57600|129600|230400,fps:7|24|24/");
4217 SetupTest();
4218 OnBitrateUpdated(kLowTargetBitrateBps);
4219
Evan Shrubsole5cd7eb82020-05-25 12:08:394220 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 09:19:054221
4222 // Insert frame, expect scaled down:
4223 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4224 InsertFrame();
4225 EXPECT_FALSE(WaitForFrame(1000));
4226 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4227 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4228
4229 // Insert frame, expect scaled down:
4230 // resolution (320x180@24fps).
4231 InsertFrame();
4232 EXPECT_FALSE(WaitForFrame(1000));
4233 EXPECT_LT(source_.sink_wants().max_pixel_count,
4234 source_.last_wants().max_pixel_count);
4235 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4236
4237 // Frame should not be dropped (min pixels per frame reached).
4238 InsertFrameAndWaitForEncoded();
4239
4240 video_stream_encoder_->Stop();
4241}
4242
4243TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334244 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324245 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:334246 "WebRTC-Video-BalancedDegradationSettings/"
4247 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044248 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334249
Åsa Persson30ab0152019-08-27 10:22:334250 const int kResolutionMinBitrateBps = 435000;
4251 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 13:13:044252 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334253
Åsa Persson45b176f2019-09-30 09:19:054254 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394255 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334256 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4257
4258 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4259 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054260 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394261 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334262 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4263
4264 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4265 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054266 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544267 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334268 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4269
4270 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4271 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054272 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544273 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:394274 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4275
Åsa Persson30ab0152019-08-27 10:22:334276 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4277 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054278 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544279 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334280 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4281
4282 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4283 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054284 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334285 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4286
4287 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044288 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334289 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054290 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544291 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334292 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4293
4294 video_stream_encoder_->Stop();
4295}
4296
Åsa Perssonccfb3402019-09-25 13:13:044297TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334298 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324299 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:334300 "WebRTC-Video-BalancedDegradationSettings/"
4301 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044302 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334303
Åsa Persson30ab0152019-08-27 10:22:334304 const int kMinBitrateBps = 425000;
4305 const int kTooLowMinBitrateBps = 424000;
4306 const int kResolutionMinBitrateBps = 435000;
4307 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 13:13:044308 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334309
Åsa Persson45b176f2019-09-30 09:19:054310 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394311 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334312 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4313
4314 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4315 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054316 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394317 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334318 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4319
4320 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4321 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054322 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544323 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334324 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4325
4326 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4327 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054328 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544329 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334330 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4331
4332 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4333 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054334 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334335 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4336
4337 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044338 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334339 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054340 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544341 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334342 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4343
4344 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044345 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334346 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054347 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334348 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4349
4350 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044351 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334352 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054353 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544354 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334355 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4356
Åsa Persson1b247f12019-08-14 15:26:394357 video_stream_encoder_->Stop();
4358}
4359
mflodmancc3d4422017-08-03 15:27:514360TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:314361 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4362 const int kWidth = 1280;
4363 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:074364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074365 DataRate::BitsPerSec(kTargetBitrateBps),
4366 DataRate::BitsPerSec(kTargetBitrateBps),
4367 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:314368
Taylor Brandstetter49fcc102018-05-16 21:20:414369 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234370 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:314371 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:514372 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414373 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:314374
Åsa Persson8c1bf952018-09-13 08:42:194375 int64_t timestamp_ms = kFrameIntervalMs;
4376 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524377 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:394378 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314379 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4380 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4381 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4382 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4383
4384 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 15:27:514385 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194386 timestamp_ms += kFrameIntervalMs;
4387 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4388 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:394389 EXPECT_THAT(source.sink_wants(),
4390 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:314391 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4392 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4393 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4394 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4395
4396 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 15:27:514397 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194398 timestamp_ms += kFrameIntervalMs;
4399 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4400 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544401 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314402 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4404 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4405 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4406
Jonathan Yubc771b72017-12-09 01:04:294407 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 15:27:514408 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194409 timestamp_ms += kFrameIntervalMs;
4410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4411 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544412 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314413 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294415 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:314416 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4417
Jonathan Yubc771b72017-12-09 01:04:294418 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 15:27:514419 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194420 timestamp_ms += kFrameIntervalMs;
4421 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4422 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544423 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294424 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 08:47:314425 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4426 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4427 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4428 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4429
Jonathan Yubc771b72017-12-09 01:04:294430 // Trigger quality adapt down, expect no change (min resolution reached).
4431 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194432 timestamp_ms += kFrameIntervalMs;
4433 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4434 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544435 EXPECT_THAT(source.sink_wants(), FpsMax());
4436 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-09 01:04:294437 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4439 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4440 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4441
Evan Shrubsole64469032020-06-11 08:45:294442 // Trigger quality adapt up, expect upscaled resolution (480x270).
4443 video_stream_encoder_->TriggerQualityHigh();
4444 timestamp_ms += kFrameIntervalMs;
4445 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4446 WaitForEncodedFrame(timestamp_ms);
4447 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4448 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4449 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4450 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4451 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4452
4453 // Trigger quality and cpu adapt up since both are most limited, expect
4454 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 10:24:334455 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294456 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194457 timestamp_ms += kFrameIntervalMs;
4458 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4459 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544460 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294461 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4462 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4463 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294464 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:294465
Evan Shrubsole64469032020-06-11 08:45:294466 // Trigger quality and cpu adapt up since both are most limited, expect
4467 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 10:24:334468 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294469 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194470 timestamp_ms += kFrameIntervalMs;
4471 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4472 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544473 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314474 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 08:45:294475 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 08:47:314476 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:294477 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4478 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314479
Evan Shrubsole64469032020-06-11 08:45:294480 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4481 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 10:24:334482 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:194483 timestamp_ms += kFrameIntervalMs;
4484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544486 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 08:47:314487 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4488 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294489 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294490 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314491
4492 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 15:27:514493 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194494 timestamp_ms += kFrameIntervalMs;
4495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524496 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:544497 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:394498 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314499 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4500 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294501 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294502 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 11:59:514503
mflodmancc3d4422017-08-03 15:27:514504 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 11:59:514505}
4506
mflodmancc3d4422017-08-03 15:27:514507TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 12:51:494508 const int kWidth = 640;
4509 const int kHeight = 360;
perkj803d97f2016-11-01 18:45:464510
Henrik Boström381d1092020-05-12 16:49:074511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074512 DataRate::BitsPerSec(kTargetBitrateBps),
4513 DataRate::BitsPerSec(kTargetBitrateBps),
4514 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:524515
perkj803d97f2016-11-01 18:45:464516 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494517 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524518 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 18:45:464519 }
4520
mflodmancc3d4422017-08-03 15:27:514521 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:464522 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494523 video_source_.IncomingCapturedFrame(CreateFrame(
4524 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524525 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 18:45:464526 }
4527
mflodmancc3d4422017-08-03 15:27:514528 video_stream_encoder_->Stop();
4529 video_stream_encoder_.reset();
perkj803d97f2016-11-01 18:45:464530 stats_proxy_.reset();
sprangf8ee65e2017-02-28 16:49:334531
Ying Wangef3998f2019-12-09 12:06:534532 EXPECT_METRIC_EQ(
4533 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4534 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 18:45:464535 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4536}
4537
mflodmancc3d4422017-08-03 15:27:514538TEST_F(VideoStreamEncoderTest,
4539 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 16:49:074540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074541 DataRate::BitsPerSec(kTargetBitrateBps),
4542 DataRate::BitsPerSec(kTargetBitrateBps),
4543 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 09:01:064544 const int kWidth = 640;
4545 const int kHeight = 360;
4546
Taylor Brandstetter49fcc102018-05-16 21:20:414547 video_stream_encoder_->SetSource(&video_source_,
4548 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 09:01:064549
4550 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4551 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524552 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 09:01:064553 }
4554
mflodmancc3d4422017-08-03 15:27:514555 video_stream_encoder_->Stop();
4556 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 09:01:064557 stats_proxy_.reset();
4558
4559 EXPECT_EQ(0,
4560 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4561}
4562
Per Kjellanderdcef6412020-10-07 13:09:054563TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4564 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034565 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:054566 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 14:24:024567
4568 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 10:32:224569 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 14:36:494570 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 13:16:284571 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4572 kDefaultFps));
sprang57c2fff2017-01-16 14:24:024573
Henrik Boström381d1092020-05-12 16:49:074574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074575 DataRate::BitsPerSec(kLowTargetBitrateBps),
4576 DataRate::BitsPerSec(kLowTargetBitrateBps),
4577 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 14:24:024578
sprang57c2fff2017-01-16 14:24:024579 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234580 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4581 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054582 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4583 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4584
Erik Språngd7329ca2019-02-21 20:19:534585 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 15:44:424586 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 12:31:234587 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024588
Per Kjellanderdcef6412020-10-07 13:09:054589 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 14:24:024590 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234591 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4592 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054593 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 12:31:234594 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024595
Per Kjellanderdcef6412020-10-07 13:09:054596 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 12:31:234597 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 09:28:414598 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 20:19:534599 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234600 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4601 WaitForEncodedFrame(CurrentTimeMs());
4602 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 20:19:534603 }
Per Kjellanderdcef6412020-10-07 13:09:054604 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 20:19:534605
mflodmancc3d4422017-08-03 15:27:514606 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 14:24:024607}
4608
Per Kjellanderf86cf4c2020-12-30 14:27:354609TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 15:53:224610 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034611 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:224612 kVideoLayersAllocation);
4613
4614 const int kDefaultFps = 30;
4615
4616 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4617 DataRate::BitsPerSec(kLowTargetBitrateBps),
4618 DataRate::BitsPerSec(kLowTargetBitrateBps),
4619 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4620
4621 video_source_.IncomingCapturedFrame(
4622 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4623 WaitForEncodedFrame(CurrentTimeMs());
4624 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4625 VideoLayersAllocation last_layer_allocation =
4626 sink_.GetLastVideoLayersAllocation();
4627 // kLowTargetBitrateBps is only enough for one spatial layer.
4628 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4629
4630 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 15:44:424631 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 15:53:224632 // Check that encoder has been updated too, not just allocation observer.
4633 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4634 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4635
Erik Språng9d69cbe2020-10-22 15:44:424636 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 15:53:224637 int number_of_layers_allocation = 1;
4638 const int64_t start_time_ms = CurrentTimeMs();
4639 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4640 video_source_.IncomingCapturedFrame(
4641 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4642 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 15:53:224643 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4644 number_of_layers_allocation = sink_.number_of_layers_allocations();
4645 VideoLayersAllocation new_allocation =
4646 sink_.GetLastVideoLayersAllocation();
4647 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4648 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4649 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4650 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4651 .target_bitrate_per_temporal_layer,
4652 last_layer_allocation.active_spatial_layers[0]
4653 .target_bitrate_per_temporal_layer);
4654 last_layer_allocation = new_allocation;
4655 }
4656 }
4657 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4658 video_stream_encoder_->Stop();
4659}
4660
4661TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134662 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354663 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4664 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4665 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354666 VideoEncoderConfig video_encoder_config;
4667 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4668 /* num_streams*/ 3, &video_encoder_config);
4669 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4670 video_encoder_config.content_type =
4671 VideoEncoderConfig::ContentType::kRealtimeVideo;
4672 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434673 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354674 VideoEncoder::GetDefaultVp8Settings());
4675 for (auto& layer : video_encoder_config.simulcast_layers) {
4676 layer.num_temporal_layers = 2;
4677 }
4678 // Simulcast layers are used for enabling/disabling streams.
4679 video_encoder_config.simulcast_layers[0].active = true;
4680 video_encoder_config.simulcast_layers[1].active = false;
4681 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034682 ConfigureEncoder(std::move(video_encoder_config),
4683 VideoStreamEncoder::BitrateAllocationCallbackType::
4684 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354685
4686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4687 DataRate::BitsPerSec(kTargetBitrateBps),
4688 DataRate::BitsPerSec(kTargetBitrateBps),
4689 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4690
4691 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4692 WaitForEncodedFrame(CurrentTimeMs());
4693 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4694 VideoLayersAllocation last_layer_allocation =
4695 sink_.GetLastVideoLayersAllocation();
4696
4697 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4698 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4699 .target_bitrate_per_temporal_layer,
4700 SizeIs(2));
4701 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4702 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4703 video_stream_encoder_->Stop();
4704}
4705
4706TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134707 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354708 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4709 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4710 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354711 VideoEncoderConfig video_encoder_config;
4712 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4713 /* num_streams*/ 3, &video_encoder_config);
4714 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4715 video_encoder_config.content_type =
4716 VideoEncoderConfig::ContentType::kRealtimeVideo;
4717 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434718 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354719 VideoEncoder::GetDefaultVp8Settings());
4720 for (auto& layer : video_encoder_config.simulcast_layers) {
4721 layer.num_temporal_layers = 2;
4722 }
4723 // Simulcast layers are used for enabling/disabling streams.
4724 video_encoder_config.simulcast_layers[0].active = true;
4725 video_encoder_config.simulcast_layers[1].active = false;
4726 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034727 ConfigureEncoder(std::move(video_encoder_config),
4728 VideoStreamEncoder::BitrateAllocationCallbackType::
4729 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354730
4731 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4732 DataRate::BitsPerSec(kTargetBitrateBps),
4733 DataRate::BitsPerSec(kTargetBitrateBps),
4734 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4735
4736 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4737 WaitForEncodedFrame(CurrentTimeMs());
4738 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4739 VideoLayersAllocation last_layer_allocation =
4740 sink_.GetLastVideoLayersAllocation();
4741
4742 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4743 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4744 .target_bitrate_per_temporal_layer,
4745 SizeIs(2));
4746 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4747
4748 video_stream_encoder_->Stop();
4749}
4750
4751TEST_F(VideoStreamEncoderTest,
4752 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4753 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4754 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354755 VideoEncoderConfig video_encoder_config;
4756 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4757 /* num_streams*/ 1, &video_encoder_config);
4758 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4759 video_encoder_config.content_type =
4760 VideoEncoderConfig::ContentType::kRealtimeVideo;
4761 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4762 vp9_settings.numberOfSpatialLayers = 2;
4763 vp9_settings.numberOfTemporalLayers = 2;
4764 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4765 vp9_settings.automaticResizeOn = false;
4766 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434767 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354768 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034769 ConfigureEncoder(std::move(video_encoder_config),
4770 VideoStreamEncoder::BitrateAllocationCallbackType::
4771 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354772
4773 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4774 DataRate::BitsPerSec(kTargetBitrateBps),
4775 DataRate::BitsPerSec(kTargetBitrateBps),
4776 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4777
4778 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4779 WaitForEncodedFrame(CurrentTimeMs());
4780 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4781 VideoLayersAllocation last_layer_allocation =
4782 sink_.GetLastVideoLayersAllocation();
4783
4784 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4785 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4786 .target_bitrate_per_temporal_layer,
4787 SizeIs(2));
4788 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4789 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4790 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4791 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4792 .target_bitrate_per_temporal_layer,
4793 SizeIs(2));
4794 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4795 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4796 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4797
4798 // Since full SVC is used, expect the top layer to utilize the full target
4799 // rate.
4800 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4801 .target_bitrate_per_temporal_layer[1],
4802 DataRate::BitsPerSec(kTargetBitrateBps));
4803 video_stream_encoder_->Stop();
4804}
4805
4806TEST_F(VideoStreamEncoderTest,
4807 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4808 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4809 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 14:27:354810 VideoEncoderConfig video_encoder_config;
4811 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4812 /* num_streams*/ 1, &video_encoder_config);
4813 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4814 video_encoder_config.content_type =
4815 VideoEncoderConfig::ContentType::kRealtimeVideo;
4816 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4817 vp9_settings.numberOfSpatialLayers = 2;
4818 vp9_settings.numberOfTemporalLayers = 2;
4819 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4820 vp9_settings.automaticResizeOn = false;
4821 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434822 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354823 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034824 ConfigureEncoder(std::move(video_encoder_config),
4825 VideoStreamEncoder::BitrateAllocationCallbackType::
4826 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354827
4828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4829 DataRate::BitsPerSec(kTargetBitrateBps),
4830 DataRate::BitsPerSec(kTargetBitrateBps),
4831 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4832
4833 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4834 WaitForEncodedFrame(CurrentTimeMs());
4835 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4836 VideoLayersAllocation last_layer_allocation =
4837 sink_.GetLastVideoLayersAllocation();
4838
4839 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4840 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4841 .target_bitrate_per_temporal_layer,
4842 SizeIs(1));
4843 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4844 .target_bitrate_per_temporal_layer,
4845 SizeIs(1));
4846 // Since full SVC is used, expect the top layer to utilize the full target
4847 // rate.
4848 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4849 .target_bitrate_per_temporal_layer[0],
4850 DataRate::BitsPerSec(kTargetBitrateBps));
4851 video_stream_encoder_->Stop();
4852}
4853
4854TEST_F(VideoStreamEncoderTest,
4855 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4856 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4857 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354858 VideoEncoderConfig video_encoder_config;
4859 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4860 /* num_streams*/ 1, &video_encoder_config);
4861 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4862 video_encoder_config.content_type =
4863 VideoEncoderConfig::ContentType::kRealtimeVideo;
4864 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4865 vp9_settings.numberOfSpatialLayers = 2;
4866 vp9_settings.numberOfTemporalLayers = 2;
4867 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4868 vp9_settings.automaticResizeOn = false;
4869 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434870 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354871 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034872 ConfigureEncoder(std::move(video_encoder_config),
4873 VideoStreamEncoder::BitrateAllocationCallbackType::
4874 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354875
4876 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4877 DataRate::BitsPerSec(kTargetBitrateBps),
4878 DataRate::BitsPerSec(kTargetBitrateBps),
4879 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4880
4881 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4882 WaitForEncodedFrame(CurrentTimeMs());
4883 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4884 VideoLayersAllocation last_layer_allocation =
4885 sink_.GetLastVideoLayersAllocation();
4886
4887 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4888 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4889 .target_bitrate_per_temporal_layer,
4890 SizeIs(2));
4891 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4892 .target_bitrate_per_temporal_layer,
4893 SizeIs(2));
4894 // Since KSVC is, spatial layers are independend except on key frames.
4895 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4896 .target_bitrate_per_temporal_layer[1],
4897 DataRate::BitsPerSec(kTargetBitrateBps));
4898 video_stream_encoder_->Stop();
4899}
4900
4901TEST_F(VideoStreamEncoderTest,
4902 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4903 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4904 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4905 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354906 VideoEncoderConfig video_encoder_config;
4907 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4908 /* num_streams*/ 1, &video_encoder_config);
4909 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4910 video_encoder_config.content_type =
4911 VideoEncoderConfig::ContentType::kRealtimeVideo;
4912 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4913 vp9_settings.numberOfSpatialLayers = 3;
4914 vp9_settings.numberOfTemporalLayers = 2;
4915 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4916 vp9_settings.automaticResizeOn = false;
4917 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434918 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354919 vp9_settings);
4920 // Simulcast layers are used for enabling/disabling streams.
4921 video_encoder_config.simulcast_layers.resize(3);
4922 video_encoder_config.simulcast_layers[0].active = false;
4923 video_encoder_config.simulcast_layers[1].active = true;
4924 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034925 ConfigureEncoder(std::move(video_encoder_config),
4926 VideoStreamEncoder::BitrateAllocationCallbackType::
4927 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354928
4929 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4930 DataRate::BitsPerSec(kTargetBitrateBps),
4931 DataRate::BitsPerSec(kTargetBitrateBps),
4932 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4933
4934 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4935 WaitForEncodedFrame(CurrentTimeMs());
4936 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4937 VideoLayersAllocation last_layer_allocation =
4938 sink_.GetLastVideoLayersAllocation();
4939
4940 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4941 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4942 .target_bitrate_per_temporal_layer,
4943 SizeIs(2));
4944 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4945 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4946
4947 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4948 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4949 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4950 .target_bitrate_per_temporal_layer,
4951 SizeIs(2));
4952 // Since full SVC is used, expect the top layer to utilize the full target
4953 // rate.
4954 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4955 .target_bitrate_per_temporal_layer[1],
4956 DataRate::BitsPerSec(kTargetBitrateBps));
4957 video_stream_encoder_->Stop();
4958}
4959
4960TEST_F(VideoStreamEncoderTest,
4961 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4962 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4963 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4964 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354965 VideoEncoderConfig video_encoder_config;
4966 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4967 /* num_streams*/ 1, &video_encoder_config);
4968 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4969 video_encoder_config.content_type =
4970 VideoEncoderConfig::ContentType::kRealtimeVideo;
4971 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4972 vp9_settings.numberOfSpatialLayers = 3;
4973 vp9_settings.numberOfTemporalLayers = 2;
4974 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4975 vp9_settings.automaticResizeOn = false;
4976 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:434977 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:354978 vp9_settings);
4979 // Simulcast layers are used for enabling/disabling streams.
4980 video_encoder_config.simulcast_layers.resize(3);
4981 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034982 ConfigureEncoder(std::move(video_encoder_config),
4983 VideoStreamEncoder::BitrateAllocationCallbackType::
4984 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354985
4986 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4987 DataRate::BitsPerSec(kTargetBitrateBps),
4988 DataRate::BitsPerSec(kTargetBitrateBps),
4989 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4990
4991 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4992 WaitForEncodedFrame(CurrentTimeMs());
4993 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4994 VideoLayersAllocation last_layer_allocation =
4995 sink_.GetLastVideoLayersAllocation();
4996
4997 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4998 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4999 .target_bitrate_per_temporal_layer,
5000 SizeIs(2));
5001 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5002 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5003
5004 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5005 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5006 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5007 .target_bitrate_per_temporal_layer,
5008 SizeIs(2));
5009 video_stream_encoder_->Stop();
5010}
5011
5012TEST_F(VideoStreamEncoderTest,
5013 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5014 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5015 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5016 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:355017 VideoEncoderConfig video_encoder_config;
5018 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5019 /* num_streams*/ 1, &video_encoder_config);
5020 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
5021 video_encoder_config.content_type =
5022 VideoEncoderConfig::ContentType::kRealtimeVideo;
5023 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5024 vp9_settings.numberOfSpatialLayers = 3;
5025 vp9_settings.numberOfTemporalLayers = 2;
5026 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5027 vp9_settings.automaticResizeOn = false;
5028 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435029 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 14:27:355030 vp9_settings);
5031 // Simulcast layers are used for enabling/disabling streams.
5032 video_encoder_config.simulcast_layers.resize(3);
5033 video_encoder_config.simulcast_layers[0].active = false;
5034 video_encoder_config.simulcast_layers[1].active = false;
5035 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:035036 ConfigureEncoder(std::move(video_encoder_config),
5037 VideoStreamEncoder::BitrateAllocationCallbackType::
5038 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:355039
5040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5041 DataRate::BitsPerSec(kTargetBitrateBps),
5042 DataRate::BitsPerSec(kTargetBitrateBps),
5043 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5044
5045 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5046 WaitForEncodedFrame(CurrentTimeMs());
5047 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5048 VideoLayersAllocation last_layer_allocation =
5049 sink_.GetLastVideoLayersAllocation();
5050
5051 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5052 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5053 .target_bitrate_per_temporal_layer,
5054 SizeIs(2));
5055 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5056 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5057 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5058 .target_bitrate_per_temporal_layer[1],
5059 DataRate::BitsPerSec(kTargetBitrateBps));
5060 video_stream_encoder_->Stop();
5061}
5062
5063TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5064 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:035065 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 14:27:355066 kVideoLayersAllocation);
5067 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5068 DataRate::BitsPerSec(kTargetBitrateBps),
5069 DataRate::BitsPerSec(kTargetBitrateBps),
5070 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5071
5072 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5073 WaitForEncodedFrame(CurrentTimeMs());
5074 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5075 VideoLayersAllocation last_layer_allocation =
5076 sink_.GetLastVideoLayersAllocation();
5077
5078 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5079 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5080 .target_bitrate_per_temporal_layer,
5081 SizeIs(1));
5082 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5083 .target_bitrate_per_temporal_layer[0],
5084 DataRate::BitsPerSec(kTargetBitrateBps));
5085 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5086 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5087 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5088 video_stream_encoder_->Stop();
5089}
5090
5091TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 15:53:225092 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5093 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035094 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:225095 kVideoLayersAllocation);
5096
5097 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5098 DataRate::BitsPerSec(kLowTargetBitrateBps),
5099 DataRate::BitsPerSec(kLowTargetBitrateBps),
5100 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5101
5102 video_source_.IncomingCapturedFrame(
5103 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5104 WaitForEncodedFrame(CurrentTimeMs());
5105 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5106 VideoLayersAllocation last_layer_allocation =
5107 sink_.GetLastVideoLayersAllocation();
5108 // kLowTargetBitrateBps is only enough for one spatial layer.
5109 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5110 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5111 .target_bitrate_per_temporal_layer[0],
5112 DataRate::BitsPerSec(kLowTargetBitrateBps));
5113
5114 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5115 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5116 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5117 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
5118 video_source_.IncomingCapturedFrame(
5119 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5120 WaitForEncodedFrame(CurrentTimeMs());
5121
5122 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5123 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5124 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5125 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5126 .target_bitrate_per_temporal_layer[0],
5127 DataRate::Zero());
5128
5129 video_stream_encoder_->Stop();
5130}
5131
Per Kjellander4190ce92020-12-15 16:24:555132TEST_F(VideoStreamEncoderTest,
5133 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5134 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035135 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 16:24:555136 kVideoLayersAllocation);
5137
5138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5139 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5140 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5141 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
5142
5143 video_source_.IncomingCapturedFrame(
5144 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5145 WaitForEncodedFrame(CurrentTimeMs());
5146 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5147 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5148 SizeIs(2));
5149 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5150 codec_width_);
5151 EXPECT_EQ(
5152 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5153 codec_height_);
5154
5155 video_source_.IncomingCapturedFrame(
5156 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5157 WaitForEncodedFrame(CurrentTimeMs());
5158 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5159 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5160 SizeIs(2));
5161 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5162 codec_width_ / 2);
5163 EXPECT_EQ(
5164 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5165 codec_height_ / 2);
5166
5167 video_stream_encoder_->Stop();
5168}
5169
Åsa Perssonc29cb2c2019-03-25 11:06:595170TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5171 // 2 TLs configured, temporal layers supported by encoder.
5172 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 13:09:055173 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035174 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055175 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595176 fake_encoder_.SetTemporalLayersSupported(0, true);
5177
5178 // Bitrate allocated across temporal layers.
5179 const int kTl0Bps = kTargetBitrateBps *
5180 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:465181 kNumTemporalLayers, /*temporal_id*/ 0,
5182 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595183 const int kTl1Bps = kTargetBitrateBps *
5184 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:465185 kNumTemporalLayers, /*temporal_id*/ 1,
5186 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595187 VideoBitrateAllocation expected_bitrate;
5188 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5189 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5190
5191 VerifyAllocatedBitrate(expected_bitrate);
5192 video_stream_encoder_->Stop();
5193}
5194
5195TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5196 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 13:09:055197 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035198 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055199 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595200 fake_encoder_.SetTemporalLayersSupported(0, false);
5201
5202 // Temporal layers not supported by the encoder.
5203 // Total bitrate should be at ti:0.
5204 VideoBitrateAllocation expected_bitrate;
5205 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
5206
5207 VerifyAllocatedBitrate(expected_bitrate);
5208 video_stream_encoder_->Stop();
5209}
5210
5211TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 13:09:055212 webrtc::test::ScopedFieldTrials field_trials(
5213 "WebRTC-Video-QualityScalerSettings/"
5214 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5215 // Reset encoder for field trials to take effect.
5216 ConfigureEncoder(video_encoder_config_.Copy());
5217
Åsa Perssonc29cb2c2019-03-25 11:06:595218 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 13:09:055219 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:035220 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:055221 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:595222 fake_encoder_.SetTemporalLayersSupported(0, true);
5223 fake_encoder_.SetTemporalLayersSupported(1, false);
5224
5225 const int kS0Bps = 150000;
5226 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:465227 kS0Bps *
5228 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5229 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595230 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:465231 kS0Bps *
5232 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5233 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:595234 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
5235 // Temporal layers not supported by si:1.
5236 VideoBitrateAllocation expected_bitrate;
5237 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5238 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5239 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5240
5241 VerifyAllocatedBitrate(expected_bitrate);
5242 video_stream_encoder_->Stop();
5243}
5244
Niels Möller7dc26b72017-12-06 09:27:485245TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5246 const int kFrameWidth = 1280;
5247 const int kFrameHeight = 720;
5248 const int kFramerate = 24;
5249
Henrik Boström381d1092020-05-12 16:49:075250 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075251 DataRate::BitsPerSec(kTargetBitrateBps),
5252 DataRate::BitsPerSec(kTargetBitrateBps),
5253 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485254 test::FrameForwarder source;
5255 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415256 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485257
5258 // Insert a single frame, triggering initial configuration.
5259 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5260 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5261
5262 EXPECT_EQ(
5263 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5264 kDefaultFramerate);
5265
5266 // Trigger reconfigure encoder (without resetting the entire instance).
5267 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515268 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5269 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 09:27:485270 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 09:27:485271 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475272 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485273 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5274
5275 // Detector should be updated with fps limit from codec config.
5276 EXPECT_EQ(
5277 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5278 kFramerate);
5279
5280 // Trigger overuse, max framerate should be reduced.
5281 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5282 stats.input_frame_rate = kFramerate;
5283 stats_proxy_->SetMockStats(stats);
5284 video_stream_encoder_->TriggerCpuOveruse();
5285 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5286 int adapted_framerate =
5287 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5288 EXPECT_LT(adapted_framerate, kFramerate);
5289
5290 // Trigger underuse, max framerate should go back to codec configured fps.
5291 // Set extra low fps, to make sure it's actually reset, not just incremented.
5292 stats = stats_proxy_->GetStats();
5293 stats.input_frame_rate = adapted_framerate / 2;
5294 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335295 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485296 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5297 EXPECT_EQ(
5298 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5299 kFramerate);
5300
5301 video_stream_encoder_->Stop();
5302}
5303
5304TEST_F(VideoStreamEncoderTest,
5305 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5306 const int kFrameWidth = 1280;
5307 const int kFrameHeight = 720;
5308 const int kLowFramerate = 15;
5309 const int kHighFramerate = 25;
5310
Henrik Boström381d1092020-05-12 16:49:075311 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075312 DataRate::BitsPerSec(kTargetBitrateBps),
5313 DataRate::BitsPerSec(kTargetBitrateBps),
5314 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485315 test::FrameForwarder source;
5316 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415317 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485318
5319 // Trigger initial configuration.
5320 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515321 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5322 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 09:27:485323 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 09:27:485324 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 06:57:515325 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 07:51:475326 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485327 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5328
5329 EXPECT_EQ(
5330 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5331 kLowFramerate);
5332
5333 // Trigger overuse, max framerate should be reduced.
5334 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5335 stats.input_frame_rate = kLowFramerate;
5336 stats_proxy_->SetMockStats(stats);
5337 video_stream_encoder_->TriggerCpuOveruse();
5338 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5339 int adapted_framerate =
5340 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5341 EXPECT_LT(adapted_framerate, kLowFramerate);
5342
5343 // Reconfigure the encoder with a new (higher max framerate), max fps should
5344 // still respect the adaptation.
Åsa Persson17107062020-10-08 06:57:515345 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 09:27:485346 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5347 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475348 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485349 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5350
5351 EXPECT_EQ(
5352 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5353 adapted_framerate);
5354
5355 // Trigger underuse, max framerate should go back to codec configured fps.
5356 stats = stats_proxy_->GetStats();
5357 stats.input_frame_rate = adapted_framerate;
5358 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335359 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485360 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5361 EXPECT_EQ(
5362 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5363 kHighFramerate);
5364
5365 video_stream_encoder_->Stop();
5366}
5367
mflodmancc3d4422017-08-03 15:27:515368TEST_F(VideoStreamEncoderTest,
5369 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 11:21:075370 const int kFrameWidth = 1280;
5371 const int kFrameHeight = 720;
5372 const int kFramerate = 24;
5373
Henrik Boström381d1092020-05-12 16:49:075374 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075375 DataRate::BitsPerSec(kTargetBitrateBps),
5376 DataRate::BitsPerSec(kTargetBitrateBps),
5377 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 11:21:075378 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:515379 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415380 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 11:21:075381
5382 // Trigger initial configuration.
5383 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515384 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5385 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 11:21:075386 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 11:21:075387 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 15:27:515388 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475389 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:515390 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 11:21:075391
Niels Möller7dc26b72017-12-06 09:27:485392 EXPECT_EQ(
5393 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5394 kFramerate);
sprangfda496a2017-06-15 11:21:075395
5396 // Trigger overuse, max framerate should be reduced.
5397 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5398 stats.input_frame_rate = kFramerate;
5399 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 15:27:515400 video_stream_encoder_->TriggerCpuOveruse();
5401 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:485402 int adapted_framerate =
5403 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 11:21:075404 EXPECT_LT(adapted_framerate, kFramerate);
5405
5406 // Change degradation preference to not enable framerate scaling. Target
5407 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 16:49:075408 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:415409 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 09:27:485410 EXPECT_EQ(
5411 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5412 kFramerate);
sprangfda496a2017-06-15 11:21:075413
mflodmancc3d4422017-08-03 15:27:515414 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 11:21:075415}
5416
mflodmancc3d4422017-08-03 15:27:515417TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495418 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075419 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075420 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5421 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5422 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495423 const int kWidth = 640;
5424 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225425
asaperssonfab67072017-04-04 12:51:495426 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225427
5428 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525429 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225430
5431 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095432 EXPECT_TRUE_WAIT(
5433 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 15:02:225434
sprangc5d62e22017-04-03 06:53:045435 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 15:02:225436
asaperssonfab67072017-04-04 12:51:495437 // Next frame is scaled.
kthelgason2bc68642017-02-07 15:02:225438 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:495439 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 15:02:225440
5441 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525442 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225443
Henrik Boström2671dac2020-05-19 14:29:095444 EXPECT_TRUE_WAIT(
5445 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 15:02:225446
mflodmancc3d4422017-08-03 15:27:515447 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225448}
5449
mflodmancc3d4422017-08-03 15:27:515450TEST_F(VideoStreamEncoderTest,
5451 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495452 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075454 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5455 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5456 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495457 const int kWidth = 640;
5458 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225459
5460 // We expect the n initial frames to get dropped.
5461 int i;
5462 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 12:51:495463 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525464 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225465 }
5466 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 12:51:495467 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525468 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 15:02:225469
5470 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 12:51:495471 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:225472
mflodmancc3d4422017-08-03 15:27:515473 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225474}
5475
mflodmancc3d4422017-08-03 15:27:515476TEST_F(VideoStreamEncoderTest,
5477 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 12:51:495478 const int kWidth = 640;
5479 const int kHeight = 360;
Henrik Boström381d1092020-05-12 16:49:075480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075481 DataRate::BitsPerSec(kLowTargetBitrateBps),
5482 DataRate::BitsPerSec(kLowTargetBitrateBps),
5483 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 15:02:225484
5485 // Set degradation preference.
mflodmancc3d4422017-08-03 15:27:515486 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415487 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 15:02:225488
asaperssonfab67072017-04-04 12:51:495489 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225490 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525491 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 15:02:225492
mflodmancc3d4422017-08-03 15:27:515493 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225494}
5495
mflodmancc3d4422017-08-03 15:27:515496TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 12:51:495497 const int kWidth = 640;
5498 const int kHeight = 360;
kthelgasonad9010c2017-02-14 08:46:515499 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:135500
5501 VideoEncoderConfig video_encoder_config;
5502 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5503 // Make format different, to force recreation of encoder.
5504 video_encoder_config.video_format.parameters["foo"] = "foo";
5505 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475506 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 16:49:075507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075508 DataRate::BitsPerSec(kLowTargetBitrateBps),
5509 DataRate::BitsPerSec(kLowTargetBitrateBps),
5510 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:185511
kthelgasonb83797b2017-02-14 19:57:255512 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 21:20:415513 video_stream_encoder_->SetSource(&video_source_,
5514 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 08:46:515515
asaperssonfab67072017-04-04 12:51:495516 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 08:46:515517 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525518 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 08:46:515519
mflodmancc3d4422017-08-03 15:27:515520 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 08:46:515521 fake_encoder_.SetQualityScaling(true);
5522}
5523
Åsa Persson139f4dc2019-08-02 07:29:585524TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5525 webrtc::test::ScopedFieldTrials field_trials(
5526 "WebRTC-Video-QualityScalerSettings/"
5527 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5528 // Reset encoder for field trials to take effect.
5529 ConfigureEncoder(video_encoder_config_.Copy());
5530 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5531 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5532 const int kWidth = 640;
5533 const int kHeight = 360;
5534
Henrik Boström381d1092020-05-12 16:49:075535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075536 DataRate::BitsPerSec(kTargetBitrateBps),
5537 DataRate::BitsPerSec(kTargetBitrateBps),
5538 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585539 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5540 // Frame should not be dropped.
5541 WaitForEncodedFrame(1);
5542
Henrik Boström381d1092020-05-12 16:49:075543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075544 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5545 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5546 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585547 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5548 // Frame should not be dropped.
5549 WaitForEncodedFrame(2);
5550
Henrik Boström381d1092020-05-12 16:49:075551 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075552 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5553 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5554 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585555 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5556 // Expect to drop this frame, the wait should time out.
5557 ExpectDroppedFrame();
5558
5559 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095560 EXPECT_TRUE_WAIT(
5561 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 07:29:585562 video_stream_encoder_->Stop();
5563}
5564
Evan Shrubsolee3da1d32020-08-14 13:58:335565TEST_F(VideoStreamEncoderTest,
5566 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5567 webrtc::test::ScopedFieldTrials field_trials(
5568 "WebRTC-Video-QualityScalerSettings/"
5569 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5570 fake_encoder_.SetQualityScaling(false);
5571 ConfigureEncoder(video_encoder_config_.Copy());
5572 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5573 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5574 const int kWidth = 640;
5575 const int kHeight = 360;
5576
5577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5578 DataRate::BitsPerSec(kTargetBitrateBps),
5579 DataRate::BitsPerSec(kTargetBitrateBps),
5580 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5581 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5582 // Frame should not be dropped.
5583 WaitForEncodedFrame(1);
5584
5585 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5586 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5587 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5588 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5589 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5590 // Frame should not be dropped.
5591 WaitForEncodedFrame(2);
5592
5593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5594 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5595 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5596 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5597 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5598 // Not dropped since quality scaling is disabled.
5599 WaitForEncodedFrame(3);
5600
5601 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 11:12:125602 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 13:58:335603 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5604
5605 video_stream_encoder_->Stop();
5606}
5607
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475608TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5609 const int kLowTargetBitrateBps = 400000;
5610 // Set simulcast.
5611 ResetEncoder("VP8", 3, 1, 1, false);
5612 fake_encoder_.SetQualityScaling(true);
5613 const int kWidth = 1280;
5614 const int kHeight = 720;
5615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5616 DataRate::BitsPerSec(kLowTargetBitrateBps),
5617 DataRate::BitsPerSec(kLowTargetBitrateBps),
5618 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5619 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5620 // Frame should not be dropped.
5621 WaitForEncodedFrame(1);
5622
5623 // Trigger QVGA "singlecast"
5624 // Update the config.
5625 VideoEncoderConfig video_encoder_config;
5626 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5627 &video_encoder_config);
Åsa Persson7f354f82021-02-04 14:52:155628 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:435629 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 14:52:155630 "VP8", /*max qp*/ 56, /*screencast*/ false,
5631 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475632 for (auto& layer : video_encoder_config.simulcast_layers) {
5633 layer.num_temporal_layers = 1;
5634 layer.max_framerate = kDefaultFramerate;
5635 }
5636 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5637 video_encoder_config.content_type =
5638 VideoEncoderConfig::ContentType::kRealtimeVideo;
5639
5640 video_encoder_config.simulcast_layers[0].active = true;
5641 video_encoder_config.simulcast_layers[1].active = false;
5642 video_encoder_config.simulcast_layers[2].active = false;
5643
5644 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5645 kMaxPayloadLength);
5646 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5647
5648 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5649 // Frame should not be dropped.
5650 WaitForEncodedFrame(2);
5651
5652 // Trigger HD "singlecast"
5653 video_encoder_config.simulcast_layers[0].active = false;
5654 video_encoder_config.simulcast_layers[1].active = false;
5655 video_encoder_config.simulcast_layers[2].active = true;
5656
5657 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5658 kMaxPayloadLength);
5659 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5660
5661 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5662 // Frame should be dropped because of initial frame drop.
5663 ExpectDroppedFrame();
5664
5665 // Expect the sink_wants to specify a scaled frame.
5666 EXPECT_TRUE_WAIT(
5667 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5668 video_stream_encoder_->Stop();
5669}
5670
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085671TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5672 const int kLowTargetBitrateBps = 400000;
5673 // Set simulcast.
5674 ResetEncoder("VP9", 1, 1, 3, false);
5675 fake_encoder_.SetQualityScaling(true);
5676 const int kWidth = 1280;
5677 const int kHeight = 720;
5678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5679 DataRate::BitsPerSec(kLowTargetBitrateBps),
5680 DataRate::BitsPerSec(kLowTargetBitrateBps),
5681 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5682 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5683 // Frame should not be dropped.
5684 WaitForEncodedFrame(1);
5685
5686 // Trigger QVGA "singlecast"
5687 // Update the config.
5688 VideoEncoderConfig video_encoder_config;
5689 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5690 &video_encoder_config);
5691 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5692 vp9_settings.numberOfSpatialLayers = 3;
5693 // Since only one layer is active - automatic resize should be enabled.
5694 vp9_settings.automaticResizeOn = true;
5695 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435696 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085697 vp9_settings);
5698 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5699 video_encoder_config.content_type =
5700 VideoEncoderConfig::ContentType::kRealtimeVideo;
5701 // Currently simulcast layers |active| flags are used to inidicate
5702 // which SVC layers are active.
5703 video_encoder_config.simulcast_layers.resize(3);
5704
5705 video_encoder_config.simulcast_layers[0].active = true;
5706 video_encoder_config.simulcast_layers[1].active = false;
5707 video_encoder_config.simulcast_layers[2].active = false;
5708
5709 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5710 kMaxPayloadLength);
5711 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5712
5713 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5714 // Frame should not be dropped.
5715 WaitForEncodedFrame(2);
5716
5717 // Trigger HD "singlecast"
5718 video_encoder_config.simulcast_layers[0].active = false;
5719 video_encoder_config.simulcast_layers[1].active = false;
5720 video_encoder_config.simulcast_layers[2].active = true;
5721
5722 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5723 kMaxPayloadLength);
5724 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5725
5726 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5727 // Frame should be dropped because of initial frame drop.
5728 ExpectDroppedFrame();
5729
5730 // Expect the sink_wants to specify a scaled frame.
5731 EXPECT_TRUE_WAIT(
5732 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5733 video_stream_encoder_->Stop();
5734}
5735
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475736TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055737 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5738 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5739 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5740 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5741 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5742 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5743 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5744 fake_encoder_.SetResolutionBitrateLimits(
5745 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5746
5747 VideoEncoderConfig video_encoder_config;
5748 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5749 &video_encoder_config);
5750 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5751 vp9_settings.numberOfSpatialLayers = 3;
5752 // Since only one layer is active - automatic resize should be enabled.
5753 vp9_settings.automaticResizeOn = true;
5754 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435755 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 08:20:055756 vp9_settings);
5757 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5758 video_encoder_config.content_type =
5759 VideoEncoderConfig::ContentType::kRealtimeVideo;
5760 // Simulcast layers are used to indicate which spatial layers are active.
5761 video_encoder_config.simulcast_layers.resize(3);
5762 video_encoder_config.simulcast_layers[0].active = false;
5763 video_encoder_config.simulcast_layers[1].active = true;
5764 video_encoder_config.simulcast_layers[2].active = false;
5765
5766 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5767 kMaxPayloadLength);
5768 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5769
5770 // The encoder bitrate limits for 360p should be used.
5771 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5772 EXPECT_FALSE(WaitForFrame(1000));
5773 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5774 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5775 VideoCodecType::kVideoCodecVP9);
5776 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5777 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5778 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5779 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5780 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5781 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5782 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5783 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5784
5785 // The encoder bitrate limits for 270p should be used.
5786 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5787 EXPECT_FALSE(WaitForFrame(1000));
5788 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5789 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5790 VideoCodecType::kVideoCodecVP9);
5791 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5792 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5793 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5794 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5795 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5796 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5797 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5798 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5799
5800 video_stream_encoder_->Stop();
5801}
5802
5803TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:515804 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5805 VideoEncoderConfig video_encoder_config;
5806 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5807 &video_encoder_config);
5808 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5809 vp9_settings.numberOfSpatialLayers = 3;
5810 // Since only one layer is active - automatic resize should be enabled.
5811 vp9_settings.automaticResizeOn = true;
5812 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435813 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 09:39:515814 vp9_settings);
5815 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5816 video_encoder_config.content_type =
5817 VideoEncoderConfig::ContentType::kRealtimeVideo;
5818 // Simulcast layers are used to indicate which spatial layers are active.
5819 video_encoder_config.simulcast_layers.resize(3);
5820 video_encoder_config.simulcast_layers[0].active = false;
5821 video_encoder_config.simulcast_layers[1].active = true;
5822 video_encoder_config.simulcast_layers[2].active = false;
5823
5824 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5825 kMaxPayloadLength);
5826 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5827
5828 // The default bitrate limits for 360p should be used.
5829 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195830 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5831 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515832 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5833 EXPECT_FALSE(WaitForFrame(1000));
5834 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5835 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5836 VideoCodecType::kVideoCodecVP9);
5837 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5838 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5839 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5840 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5841 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5842 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5843 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5844 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5845
5846 // The default bitrate limits for 270p should be used.
5847 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 12:29:195848 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5849 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:515850 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5851 EXPECT_FALSE(WaitForFrame(1000));
5852 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5853 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5854 VideoCodecType::kVideoCodecVP9);
5855 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5856 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5857 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5858 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5859 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5860 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5861 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5862 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5863
5864 video_stream_encoder_->Stop();
5865}
5866
5867TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5868 webrtc::test::ScopedFieldTrials field_trials(
5869 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5870 VideoEncoderConfig video_encoder_config;
5871 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5872 &video_encoder_config);
5873 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5874 vp9_settings.numberOfSpatialLayers = 3;
5875 // Since only one layer is active - automatic resize should be enabled.
5876 vp9_settings.automaticResizeOn = true;
5877 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435878 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 09:39:515879 vp9_settings);
5880 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5881 video_encoder_config.content_type =
5882 VideoEncoderConfig::ContentType::kRealtimeVideo;
5883 // Simulcast layers are used to indicate which spatial layers are active.
5884 video_encoder_config.simulcast_layers.resize(3);
5885 video_encoder_config.simulcast_layers[0].active = false;
5886 video_encoder_config.simulcast_layers[1].active = true;
5887 video_encoder_config.simulcast_layers[2].active = false;
5888
5889 // Reset encoder for field trials to take effect.
5890 ConfigureEncoder(video_encoder_config.Copy());
5891
5892 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5893 kMaxPayloadLength);
5894 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5895
5896 // The default bitrate limits for 360p should not be used.
5897 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195898 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5899 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515900 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5901 EXPECT_FALSE(WaitForFrame(1000));
5902 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 12:29:195903 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 09:39:515904 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5905 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5906 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5907 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5908 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5909 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5910
5911 video_stream_encoder_->Stop();
5912}
5913
5914TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5915 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5916 /*num_spatial_layers=*/1, /*screenshare=*/false);
5917
5918 // The default singlecast bitrate limits for 720p should not be used.
5919 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 12:29:195920 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5921 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 09:39:515922 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5923 EXPECT_FALSE(WaitForFrame(1000));
5924 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5925 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5926 VideoCodecType::kVideoCodecVP9);
5927 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5928 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5929 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5930 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5931 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5932 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5933
5934 video_stream_encoder_->Stop();
5935}
5936
5937TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055938 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5939 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5940 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5941 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5942 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5943 fake_encoder_.SetResolutionBitrateLimits(
5944 {kEncoderLimits180p, kEncoderLimits720p});
5945
5946 VideoEncoderConfig video_encoder_config;
5947 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5948 &video_encoder_config);
5949 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5950 vp9_settings.numberOfSpatialLayers = 3;
5951 // Since only one layer is active - automatic resize should be enabled.
5952 vp9_settings.automaticResizeOn = true;
5953 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 17:21:435954 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 08:20:055955 vp9_settings);
5956 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5957 video_encoder_config.content_type =
5958 VideoEncoderConfig::ContentType::kRealtimeVideo;
5959 // Simulcast layers are used to indicate which spatial layers are active.
5960 video_encoder_config.simulcast_layers.resize(3);
5961 video_encoder_config.simulcast_layers[0].active = true;
5962 video_encoder_config.simulcast_layers[1].active = false;
5963 video_encoder_config.simulcast_layers[2].active = false;
5964
5965 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5966 kMaxPayloadLength);
5967 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5968
5969 // Limits not applied on lowest stream, limits for 180p should not be used.
5970 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5971 EXPECT_FALSE(WaitForFrame(1000));
5972 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5973 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5974 VideoCodecType::kVideoCodecVP9);
5975 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5976 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5977 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5978 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5979 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5980 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5981 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5982 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5983
5984 video_stream_encoder_->Stop();
5985}
5986
5987TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475988 InitialFrameDropActivatesWhenResolutionIncreases) {
5989 const int kWidth = 640;
5990 const int kHeight = 360;
5991
5992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5993 DataRate::BitsPerSec(kTargetBitrateBps),
5994 DataRate::BitsPerSec(kTargetBitrateBps),
5995 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5996 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5997 // Frame should not be dropped.
5998 WaitForEncodedFrame(1);
5999
6000 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6001 DataRate::BitsPerSec(kLowTargetBitrateBps),
6002 DataRate::BitsPerSec(kLowTargetBitrateBps),
6003 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
6004 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6005 // Frame should not be dropped, bitrate not too low for frame.
6006 WaitForEncodedFrame(2);
6007
6008 // Incoming resolution increases.
6009 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6010 // Expect to drop this frame, bitrate too low for frame.
6011 ExpectDroppedFrame();
6012
6013 // Expect the sink_wants to specify a scaled frame.
6014 EXPECT_TRUE_WAIT(
6015 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6016 video_stream_encoder_->Stop();
6017}
6018
6019TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6020 const int kWidth = 640;
6021 const int kHeight = 360;
6022 // So that quality scaling doesn't happen by itself.
6023 fake_encoder_.SetQp(kQpHigh);
6024
6025 AdaptingFrameForwarder source(&time_controller_);
6026 source.set_adaptation_enabled(true);
6027 video_stream_encoder_->SetSource(
6028 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6029
6030 int timestamp = 1;
6031
6032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6033 DataRate::BitsPerSec(kTargetBitrateBps),
6034 DataRate::BitsPerSec(kTargetBitrateBps),
6035 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6036 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6037 WaitForEncodedFrame(timestamp);
6038 timestamp += 9000;
6039 // Long pause to disable all first BWE drop logic.
6040 AdvanceTime(TimeDelta::Millis(1000));
6041
6042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6043 DataRate::BitsPerSec(kLowTargetBitrateBps),
6044 DataRate::BitsPerSec(kLowTargetBitrateBps),
6045 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
6046 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6047 // Not dropped frame, as initial frame drop is disabled by now.
6048 WaitForEncodedFrame(timestamp);
6049 timestamp += 9000;
6050 AdvanceTime(TimeDelta::Millis(100));
6051
6052 // Quality adaptation down.
6053 video_stream_encoder_->TriggerQualityLow();
6054
6055 // Adaptation has an effect.
6056 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6057 5000);
6058
6059 // Frame isn't dropped as initial frame dropper is disabled.
6060 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6061 WaitForEncodedFrame(timestamp);
6062 timestamp += 9000;
6063 AdvanceTime(TimeDelta::Millis(100));
6064
6065 // Quality adaptation up.
6066 video_stream_encoder_->TriggerQualityHigh();
6067
6068 // Adaptation has an effect.
6069 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6070 5000);
6071
6072 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6073 // Frame should not be dropped, as initial framedropper is off.
6074 WaitForEncodedFrame(timestamp);
6075
6076 video_stream_encoder_->Stop();
6077}
6078
Åsa Persson7f354f82021-02-04 14:52:156079TEST_F(VideoStreamEncoderTest,
6080 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6081 const int kMinStartBps360p = 222000;
6082 fake_encoder_.SetResolutionBitrateLimits(
6083 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6084 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6085 800000)});
6086
6087 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6088 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6089 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6090 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6091 0, 0, 0);
6092 // Frame should not be dropped, bitrate not too low for frame.
6093 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6094 WaitForEncodedFrame(1);
6095
6096 // Incoming resolution increases, initial frame drop activates.
6097 // Frame should be dropped, link allocation too low for frame.
6098 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6099 ExpectDroppedFrame();
6100
6101 // Expect sink_wants to specify a scaled frame.
6102 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6103 5000);
6104 video_stream_encoder_->Stop();
6105}
6106
6107TEST_F(VideoStreamEncoderTest,
6108 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6109 const int kMinStartBps360p = 222000;
6110 fake_encoder_.SetResolutionBitrateLimits(
6111 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6112 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6113 800000)});
6114
6115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6116 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6117 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6118 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6119 0, 0, 0);
6120 // Frame should not be dropped, bitrate not too low for frame.
6121 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6122 WaitForEncodedFrame(1);
6123
6124 // Incoming resolution increases, initial frame drop activates.
6125 // Frame should be dropped, link allocation not too low for frame.
6126 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6127 WaitForEncodedFrame(2);
6128
6129 video_stream_encoder_->Stop();
6130}
6131
Åsa Perssone644a032019-11-08 14:56:006132TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6133 webrtc::test::ScopedFieldTrials field_trials(
6134 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
6135
6136 // Reset encoder for field trials to take effect.
6137 VideoEncoderConfig config = video_encoder_config_.Copy();
6138 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 09:34:326139 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 14:56:006140 ConfigureEncoder(std::move(config));
6141 fake_encoder_.SetQp(kQpLow);
6142
6143 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 12:31:236144 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 14:56:006145 source.set_adaptation_enabled(true);
6146 video_stream_encoder_->SetSource(&source,
6147 DegradationPreference::MAINTAIN_FRAMERATE);
6148
6149 // Start at low bitrate.
6150 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 16:49:076151 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6152 DataRate::BitsPerSec(kLowBitrateBps),
6153 DataRate::BitsPerSec(kLowBitrateBps),
6154 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:006155
6156 // Expect first frame to be dropped and resolution to be limited.
6157 const int kWidth = 1280;
6158 const int kHeight = 720;
6159 const int64_t kFrameIntervalMs = 100;
6160 int64_t timestamp_ms = kFrameIntervalMs;
6161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6162 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:096163 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6164 5000);
Åsa Perssone644a032019-11-08 14:56:006165
6166 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 16:49:076167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6168 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:006169
6170 // Insert frames and advance |min_duration_ms|.
6171 for (size_t i = 1; i <= 10; i++) {
6172 timestamp_ms += kFrameIntervalMs;
6173 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6174 WaitForEncodedFrame(timestamp_ms);
6175 }
6176 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6177 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6178
Tomas Gunnarsson612445e2020-09-21 12:31:236179 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 14:56:006180
6181 // Insert frame should trigger high BW and release quality limitation.
6182 timestamp_ms += kFrameIntervalMs;
6183 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6184 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 16:49:076185 // The ramp-up code involves the adaptation queue, give it time to execute.
6186 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 11:12:126187 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 12:08:396188 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 14:56:006189
6190 // Frame should not be adapted.
6191 timestamp_ms += kFrameIntervalMs;
6192 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6193 WaitForEncodedFrame(kWidth, kHeight);
6194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6195
6196 video_stream_encoder_->Stop();
6197}
6198
mflodmancc3d4422017-08-03 15:27:516199TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 11:12:286200 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 16:19:316201 webrtc::test::ScopedFieldTrials field_trials(
6202 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 12:31:236203 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 11:12:286204 source.set_adaptation_enabled(true);
6205 video_stream_encoder_->SetSource(&source,
6206 DegradationPreference::MAINTAIN_FRAMERATE);
6207 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6208 DataRate::BitsPerSec(kTargetBitrateBps),
6209 DataRate::BitsPerSec(kTargetBitrateBps),
6210 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6211 fake_encoder_.SetQp(kQpHigh + 1);
6212 const int kWidth = 1280;
6213 const int kHeight = 720;
6214 const int64_t kFrameIntervalMs = 100;
6215 int64_t timestamp_ms = kFrameIntervalMs;
6216 for (size_t i = 1; i <= 100; i++) {
6217 timestamp_ms += kFrameIntervalMs;
6218 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6219 WaitForEncodedFrame(timestamp_ms);
6220 }
6221 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6222 // for the first time.
6223 // TODO(eshr): We should avoid these waits by using threads with simulated
6224 // time.
6225 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6226 2000 * 2.5 * 2);
6227 timestamp_ms += kFrameIntervalMs;
6228 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6229 WaitForEncodedFrame(timestamp_ms);
6230 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6231 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6232 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6233
6234 // Disable Quality scaling by turning off scaler on the encoder and
6235 // reconfiguring.
6236 fake_encoder_.SetQualityScaling(false);
6237 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6238 kMaxPayloadLength);
6239 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 12:31:236240 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 11:12:286241 // Since we turned off the quality scaler, the adaptations made by it are
6242 // removed.
6243 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6245
6246 video_stream_encoder_->Stop();
6247}
6248
6249TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:316250 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6251 const int kTooSmallWidth = 10;
6252 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 16:49:076253 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076254 DataRate::BitsPerSec(kTargetBitrateBps),
6255 DataRate::BitsPerSec(kTargetBitrateBps),
6256 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:316257
Taylor Brandstetter49fcc102018-05-16 21:20:416258 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:316259 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:516260 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416261 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 12:08:396262 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 08:47:316263 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6264
6265 // Trigger adapt down, too small frame, expect no change.
6266 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526267 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516268 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:396269 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:316270 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6271 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6272
mflodmancc3d4422017-08-03 15:27:516273 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:316274}
6275
mflodmancc3d4422017-08-03 15:27:516276TEST_F(VideoStreamEncoderTest,
6277 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226278 const int kTooSmallWidth = 10;
6279 const int kTooSmallHeight = 10;
6280 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 16:49:076281 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076282 DataRate::BitsPerSec(kTargetBitrateBps),
6283 DataRate::BitsPerSec(kTargetBitrateBps),
6284 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226285
Taylor Brandstetter49fcc102018-05-16 21:20:416286 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:226287 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:416288 video_stream_encoder_->SetSource(&source,
6289 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:396290 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:226291 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6292 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6293
6294 // Trigger adapt down, expect limited framerate.
6295 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526296 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516297 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396298 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226299 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6300 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6301 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6302
6303 // Trigger adapt down, too small frame, expect no change.
6304 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526305 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:516306 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396307 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226308 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6309 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6310 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6311
mflodmancc3d4422017-08-03 15:27:516312 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226313}
6314
mflodmancc3d4422017-08-03 15:27:516315TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 08:12:526316 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 16:49:076317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076318 DataRate::BitsPerSec(kTargetBitrateBps),
6319 DataRate::BitsPerSec(kTargetBitrateBps),
6320 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 07:51:476321 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 08:12:526322 const int kFrameWidth = 1280;
6323 const int kFrameHeight = 720;
6324 video_source_.IncomingCapturedFrame(
6325 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526326 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 15:27:516327 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:526328}
6329
sprangb1ca0732017-02-01 16:38:126330// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 15:27:516331TEST_F(VideoStreamEncoderTest,
6332 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 16:49:076333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076334 DataRate::BitsPerSec(kTargetBitrateBps),
6335 DataRate::BitsPerSec(kTargetBitrateBps),
6336 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 16:38:126337
6338 const int kFrameWidth = 1280;
6339 const int kFrameHeight = 720;
6340 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 15:27:516341 // requested by
6342 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 16:38:126343 video_source_.set_adaptation_enabled(true);
6344
6345 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196346 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526347 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126348
6349 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516350 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 16:38:126351 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196352 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526353 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 16:38:126354
asaperssonfab67072017-04-04 12:51:496355 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 10:24:336356 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 16:38:126357 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196358 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526359 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126360
mflodmancc3d4422017-08-03 15:27:516361 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 16:38:126362}
sprangfe627f32017-03-29 15:24:596363
mflodmancc3d4422017-08-03 15:27:516364TEST_F(VideoStreamEncoderTest,
6365 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-03 06:53:046366 const int kFrameWidth = 1280;
6367 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:046368
Henrik Boström381d1092020-05-12 16:49:076369 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076370 DataRate::BitsPerSec(kTargetBitrateBps),
6371 DataRate::BitsPerSec(kTargetBitrateBps),
6372 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516373 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416374 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046375 video_source_.set_adaptation_enabled(true);
6376
Tomas Gunnarsson612445e2020-09-21 12:31:236377 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046378
6379 video_source_.IncomingCapturedFrame(
6380 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526381 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046382
6383 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 15:27:516384 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046385
6386 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 14:06:526387 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046388 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046389 video_source_.IncomingCapturedFrame(
6390 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526391 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046392 }
6393
6394 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 15:27:516395 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046396 int num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526397 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046398 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046399 video_source_.IncomingCapturedFrame(
6400 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526401 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046402 ++num_frames_dropped;
6403 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566404 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046405 }
6406 }
6407
sprang4847ae62017-06-27 14:06:526408 // Add some slack to account for frames dropped by the frame dropper.
6409 const int kErrorMargin = 1;
6410 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046411 kErrorMargin);
6412
6413 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 15:27:516414 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046415 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 08:42:196416 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046417 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046418 video_source_.IncomingCapturedFrame(
6419 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526420 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046421 ++num_frames_dropped;
6422 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566423 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046424 }
6425 }
sprang4847ae62017-06-27 14:06:526426 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-03 06:53:046427 kErrorMargin);
6428
6429 // Go back up one step.
Henrik Boström91aa7322020-04-28 10:24:336430 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046431 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526432 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046433 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046434 video_source_.IncomingCapturedFrame(
6435 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526436 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046437 ++num_frames_dropped;
6438 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566439 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046440 }
6441 }
sprang4847ae62017-06-27 14:06:526442 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046443 kErrorMargin);
6444
6445 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 10:24:336446 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046447 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526448 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046449 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046450 video_source_.IncomingCapturedFrame(
6451 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526452 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046453 ++num_frames_dropped;
6454 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566455 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046456 }
6457 }
6458 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6459
mflodmancc3d4422017-08-03 15:27:516460 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046461}
6462
mflodmancc3d4422017-08-03 15:27:516463TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-03 06:53:046464 const int kFramerateFps = 5;
6465 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-03 06:53:046466 const int kFrameWidth = 1280;
6467 const int kFrameHeight = 720;
6468
sprang4847ae62017-06-27 14:06:526469 // Reconfigure encoder with two temporal layers and screensharing, which will
6470 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 07:51:476471 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 14:06:526472
Henrik Boström381d1092020-05-12 16:49:076473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076474 DataRate::BitsPerSec(kTargetBitrateBps),
6475 DataRate::BitsPerSec(kTargetBitrateBps),
6476 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516477 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416478 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046479 video_source_.set_adaptation_enabled(true);
6480
Tomas Gunnarsson612445e2020-09-21 12:31:236481 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046482
6483 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-09 01:04:296484 rtc::VideoSinkWants last_wants;
6485 do {
6486 last_wants = video_source_.sink_wants();
6487
sprangc5d62e22017-04-03 06:53:046488 // Insert frames to get a new fps estimate...
6489 for (int j = 0; j < kFramerateFps; ++j) {
6490 video_source_.IncomingCapturedFrame(
6491 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-09 01:04:296492 if (video_source_.last_sent_width()) {
6493 sink_.WaitForEncodedFrame(timestamp_ms);
6494 }
sprangc5d62e22017-04-03 06:53:046495 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:236496 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-03 06:53:046497 }
6498 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 15:27:516499 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:296500 } while (video_source_.sink_wants().max_framerate_fps <
6501 last_wants.max_framerate_fps);
sprangc5d62e22017-04-03 06:53:046502
Evan Shrubsole5cd7eb82020-05-25 12:08:396503 EXPECT_THAT(video_source_.sink_wants(),
6504 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-14 06:25:226505
mflodmancc3d4422017-08-03 15:27:516506 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046507}
asaperssonf7e294d2017-06-14 06:25:226508
mflodmancc3d4422017-08-03 15:27:516509TEST_F(VideoStreamEncoderTest,
6510 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226511 const int kWidth = 1280;
6512 const int kHeight = 720;
6513 const int64_t kFrameIntervalMs = 150;
6514 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076515 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076516 DataRate::BitsPerSec(kTargetBitrateBps),
6517 DataRate::BitsPerSec(kTargetBitrateBps),
6518 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226519
Taylor Brandstetter49fcc102018-05-16 21:20:416520 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236521 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226522 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416523 video_stream_encoder_->SetSource(&source,
6524 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226525 timestamp_ms += kFrameIntervalMs;
6526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526527 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396528 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6531 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6532
6533 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516534 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226535 timestamp_ms += kFrameIntervalMs;
6536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526537 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396538 EXPECT_THAT(source.sink_wants(),
6539 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226540 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6541 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6542 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6543
6544 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516545 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226546 timestamp_ms += kFrameIntervalMs;
6547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526548 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546549 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226550 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6551 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6552 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6553
6554 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516555 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226556 timestamp_ms += kFrameIntervalMs;
6557 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526558 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546559 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6562 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6563
6564 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516565 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226566 timestamp_ms += kFrameIntervalMs;
6567 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526568 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546569 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226570 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6571 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6572 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6573
6574 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516575 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226576 timestamp_ms += kFrameIntervalMs;
6577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526578 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546579 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6582 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6583
6584 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516585 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226586 timestamp_ms += kFrameIntervalMs;
6587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526588 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546589 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6592 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6593
6594 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 15:27:516595 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226596 timestamp_ms += kFrameIntervalMs;
6597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526598 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546599 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226600 rtc::VideoSinkWants last_wants = source.sink_wants();
6601 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6602 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6603 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6604
6605 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 15:27:516606 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226607 timestamp_ms += kFrameIntervalMs;
6608 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526609 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546610 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-14 06:25:226611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6612 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6613 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6614
Åsa Perssonf5f7e8e2021-06-09 20:55:386615 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516616 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226617 timestamp_ms += kFrameIntervalMs;
6618 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526619 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546620 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226621 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6622 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6623 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6624
6625 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516626 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226627 timestamp_ms += kFrameIntervalMs;
6628 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526629 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546630 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6632 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6633 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6634
6635 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516636 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226637 timestamp_ms += kFrameIntervalMs;
6638 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526639 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546640 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226641 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6643 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6644
6645 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516646 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226647 timestamp_ms += kFrameIntervalMs;
6648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526649 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546650 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226651 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6652 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6653 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6654
6655 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516656 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226657 timestamp_ms += kFrameIntervalMs;
6658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526659 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546660 EXPECT_THAT(source.sink_wants(), FpsMax());
6661 EXPECT_EQ(source.sink_wants().max_pixel_count,
6662 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-14 06:25:226663 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6664 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6665 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6666
6667 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516668 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226669 timestamp_ms += kFrameIntervalMs;
6670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526671 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546672 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226673 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6675 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6676
Åsa Persson30ab0152019-08-27 10:22:336677 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516678 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226679 timestamp_ms += kFrameIntervalMs;
6680 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526681 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546682 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396683 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226684 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6685 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6686 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6687
6688 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516689 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396690 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226691 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6692
mflodmancc3d4422017-08-03 15:27:516693 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226694}
6695
mflodmancc3d4422017-08-03 15:27:516696TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-14 06:25:226697 const int kWidth = 1280;
6698 const int kHeight = 720;
6699 const int64_t kFrameIntervalMs = 150;
6700 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076701 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076702 DataRate::BitsPerSec(kTargetBitrateBps),
6703 DataRate::BitsPerSec(kTargetBitrateBps),
6704 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226705
Taylor Brandstetter49fcc102018-05-16 21:20:416706 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236707 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226708 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416709 video_stream_encoder_->SetSource(&source,
6710 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226711 timestamp_ms += kFrameIntervalMs;
6712 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526713 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396714 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226715 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6716 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6717 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6719 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6720 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6721
6722 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516723 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226724 timestamp_ms += kFrameIntervalMs;
6725 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526726 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396727 EXPECT_THAT(source.sink_wants(),
6728 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226729 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6731 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6732 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6733 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6734 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6735
6736 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516737 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226738 timestamp_ms += kFrameIntervalMs;
6739 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526740 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546741 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226742 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6743 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6744 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6745 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6746 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6747 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6748
6749 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516750 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226751 timestamp_ms += kFrameIntervalMs;
6752 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526753 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546754 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 08:45:296755 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-14 06:25:226756 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6757 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6758 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6759 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6760 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6761
Evan Shrubsole64469032020-06-11 08:45:296762 // Trigger cpu adapt up, expect no change since QP is most limited.
6763 {
6764 // Store current sink wants since we expect no change and if there is no
6765 // change then last_wants() is not updated.
6766 auto previous_sink_wants = source.sink_wants();
6767 video_stream_encoder_->TriggerCpuUnderuse();
6768 timestamp_ms += kFrameIntervalMs;
6769 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6770 WaitForEncodedFrame(timestamp_ms);
6771 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6772 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6773 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6774 }
6775
6776 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6777 video_stream_encoder_->TriggerQualityHigh();
6778 timestamp_ms += kFrameIntervalMs;
6779 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6780 WaitForEncodedFrame(timestamp_ms);
6781 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6782 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6783 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6784 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6786 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6787 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6788
6789 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6790 // expect increased resolution (960x540@30fps).
6791 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336792 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226793 timestamp_ms += kFrameIntervalMs;
6794 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526795 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 08:45:296796 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226797 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6798 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6799 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6800 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6801 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296802 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226803
Evan Shrubsole64469032020-06-11 08:45:296804 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6805 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516806 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336807 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226808 timestamp_ms += kFrameIntervalMs;
6809 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526810 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546811 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396812 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226813 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6814 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6815 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6816 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6817 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296818 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226819
6820 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516821 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396822 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226823 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296824 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226825
mflodmancc3d4422017-08-03 15:27:516826 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226827}
6828
mflodmancc3d4422017-08-03 15:27:516829TEST_F(VideoStreamEncoderTest,
6830 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-14 06:25:226831 const int kWidth = 640;
6832 const int kHeight = 360;
6833 const int kFpsLimit = 15;
6834 const int64_t kFrameIntervalMs = 150;
6835 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076837 DataRate::BitsPerSec(kTargetBitrateBps),
6838 DataRate::BitsPerSec(kTargetBitrateBps),
6839 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226840
Taylor Brandstetter49fcc102018-05-16 21:20:416841 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236842 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226843 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416844 video_stream_encoder_->SetSource(&source,
6845 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226846 timestamp_ms += kFrameIntervalMs;
6847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526848 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396849 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226850 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6851 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6852 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6854 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6855 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6856
6857 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516858 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226859 timestamp_ms += kFrameIntervalMs;
6860 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526861 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396862 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226863 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6865 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6866 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6867 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6868 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6869
6870 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516871 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226872 timestamp_ms += kFrameIntervalMs;
6873 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526874 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546875 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226876 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296877 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:226878 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6879 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6880 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6881 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6882
Evan Shrubsole64469032020-06-11 08:45:296883 // Trigger cpu adapt up, expect no change because quality is most limited.
6884 {
6885 auto previous_sink_wants = source.sink_wants();
6886 // Store current sink wants since we expect no change ind if there is no
6887 // change then last__wants() is not updated.
6888 video_stream_encoder_->TriggerCpuUnderuse();
6889 timestamp_ms += kFrameIntervalMs;
6890 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6891 WaitForEncodedFrame(timestamp_ms);
6892 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6893 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6894 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6895 }
6896
6897 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6898 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226899 timestamp_ms += kFrameIntervalMs;
6900 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526901 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546902 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226903 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6904 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6905 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296906 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6907 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6908 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226909
Evan Shrubsole64469032020-06-11 08:45:296910 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516911 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 08:45:296912 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226913 timestamp_ms += kFrameIntervalMs;
6914 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526915 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396916 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226917 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6918 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6919 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6920 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6921 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296922 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226923
6924 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516925 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396926 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226927 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296928 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226929
mflodmancc3d4422017-08-03 15:27:516930 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226931}
6932
mflodmancc3d4422017-08-03 15:27:516933TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 13:53:486934 const int kFrameWidth = 1920;
6935 const int kFrameHeight = 1080;
6936 // 3/4 of 1920.
6937 const int kAdaptedFrameWidth = 1440;
6938 // 3/4 of 1080 rounded down to multiple of 4.
6939 const int kAdaptedFrameHeight = 808;
6940 const int kFramerate = 24;
6941
Henrik Boström381d1092020-05-12 16:49:076942 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076943 DataRate::BitsPerSec(kTargetBitrateBps),
6944 DataRate::BitsPerSec(kTargetBitrateBps),
6945 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 13:53:486946 // Trigger reconfigure encoder (without resetting the entire instance).
6947 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 10:57:586948 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6949 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 13:53:486950 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 13:53:486951 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:436952 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 15:27:516953 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:476954 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:516955 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 13:53:486956
6957 video_source_.set_adaptation_enabled(true);
6958
6959 video_source_.IncomingCapturedFrame(
6960 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526961 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 13:53:486962
6963 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516964 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 13:53:486965 video_source_.IncomingCapturedFrame(
6966 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526967 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 13:53:486968
mflodmancc3d4422017-08-03 15:27:516969 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 13:53:486970}
6971
mflodmancc3d4422017-08-03 15:27:516972TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 14:06:526973 const int kFrameWidth = 1280;
6974 const int kFrameHeight = 720;
6975 const int kLowFps = 2;
6976 const int kHighFps = 30;
6977
Henrik Boström381d1092020-05-12 16:49:076978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076979 DataRate::BitsPerSec(kTargetBitrateBps),
6980 DataRate::BitsPerSec(kTargetBitrateBps),
6981 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:526982
Tomas Gunnarsson612445e2020-09-21 12:31:236983 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:526984 max_framerate_ = kLowFps;
6985
6986 // Insert 2 seconds of 2fps video.
6987 for (int i = 0; i < kLowFps * 2; ++i) {
6988 video_source_.IncomingCapturedFrame(
6989 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6990 WaitForEncodedFrame(timestamp_ms);
6991 timestamp_ms += 1000 / kLowFps;
6992 }
6993
6994 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 16:49:076995 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076996 DataRate::BitsPerSec(kTargetBitrateBps),
6997 DataRate::BitsPerSec(kTargetBitrateBps),
6998 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:526999 video_source_.IncomingCapturedFrame(
7000 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7001 WaitForEncodedFrame(timestamp_ms);
7002 timestamp_ms += 1000 / kLowFps;
7003
7004 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
7005
7006 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 08:48:487007 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 14:06:527008 const int kFrameIntervalMs = 1000 / kHighFps;
7009 max_framerate_ = kHighFps;
7010 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
7011 video_source_.IncomingCapturedFrame(
7012 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7013 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
7014 // be dropped if the encoder hans't been updated with the new higher target
7015 // framerate yet, causing it to overshoot the target bitrate and then
7016 // suffering the wrath of the media optimizer.
7017 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
7018 timestamp_ms += kFrameIntervalMs;
7019 }
7020
7021 // Don expect correct measurement just yet, but it should be higher than
7022 // before.
7023 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7024
mflodmancc3d4422017-08-03 15:27:517025 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:527026}
7027
mflodmancc3d4422017-08-03 15:27:517028TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 14:06:527029 const int kFrameWidth = 1280;
7030 const int kFrameHeight = 720;
7031 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 13:09:057032 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:037033 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:057034 kVideoBitrateAllocation);
sprang4847ae62017-06-27 14:06:527035
Henrik Boström381d1092020-05-12 16:49:077036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077037 DataRate::BitsPerSec(kTargetBitrateBps),
7038 DataRate::BitsPerSec(kTargetBitrateBps),
7039 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:517040 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 14:06:527041
7042 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 12:31:237043 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:527044 video_source_.IncomingCapturedFrame(
7045 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7046 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 13:09:057047 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:527048
7049 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 16:49:077050 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7051 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
7052 0, 1, 0);
sprang4847ae62017-06-27 14:06:527053
7054 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 08:48:487055 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:237056 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 14:06:527057
Per Kjellanderdcef6412020-10-07 13:09:057058 // No more allocations has been made.
sprang4847ae62017-06-27 14:06:527059 video_source_.IncomingCapturedFrame(
7060 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7061 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 13:09:057062 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:527063
mflodmancc3d4422017-08-03 15:27:517064 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:527065}
ilnik6b826ef2017-06-16 13:53:487066
Niels Möller4db138e2018-04-19 07:04:137067TEST_F(VideoStreamEncoderTest,
7068 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7069 const int kFrameWidth = 1280;
7070 const int kFrameHeight = 720;
7071 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 16:49:077072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077073 DataRate::BitsPerSec(kTargetBitrateBps),
7074 DataRate::BitsPerSec(kTargetBitrateBps),
7075 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:137076 video_source_.IncomingCapturedFrame(
7077 CreateFrame(1, kFrameWidth, kFrameHeight));
7078 WaitForEncodedFrame(1);
7079 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7080 .low_encode_usage_threshold_percent,
7081 default_options.low_encode_usage_threshold_percent);
7082 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7083 .high_encode_usage_threshold_percent,
7084 default_options.high_encode_usage_threshold_percent);
7085 video_stream_encoder_->Stop();
7086}
7087
7088TEST_F(VideoStreamEncoderTest,
7089 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7090 const int kFrameWidth = 1280;
7091 const int kFrameHeight = 720;
7092 CpuOveruseOptions hardware_options;
7093 hardware_options.low_encode_usage_threshold_percent = 150;
7094 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 11:42:187095 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 07:04:137096
Henrik Boström381d1092020-05-12 16:49:077097 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077098 DataRate::BitsPerSec(kTargetBitrateBps),
7099 DataRate::BitsPerSec(kTargetBitrateBps),
7100 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:137101 video_source_.IncomingCapturedFrame(
7102 CreateFrame(1, kFrameWidth, kFrameHeight));
7103 WaitForEncodedFrame(1);
7104 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7105 .low_encode_usage_threshold_percent,
7106 hardware_options.low_encode_usage_threshold_percent);
7107 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7108 .high_encode_usage_threshold_percent,
7109 hardware_options.high_encode_usage_threshold_percent);
7110 video_stream_encoder_->Stop();
7111}
7112
Jakob Ivarsson461b1d92021-01-22 15:27:437113TEST_F(VideoStreamEncoderTest,
7114 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7115 const int kFrameWidth = 1280;
7116 const int kFrameHeight = 720;
7117
7118 const CpuOveruseOptions default_options;
7119 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7120 DataRate::BitsPerSec(kTargetBitrateBps),
7121 DataRate::BitsPerSec(kTargetBitrateBps),
7122 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7123 video_source_.IncomingCapturedFrame(
7124 CreateFrame(1, kFrameWidth, kFrameHeight));
7125 WaitForEncodedFrame(1);
7126 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7127 .low_encode_usage_threshold_percent,
7128 default_options.low_encode_usage_threshold_percent);
7129 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7130 .high_encode_usage_threshold_percent,
7131 default_options.high_encode_usage_threshold_percent);
7132
7133 CpuOveruseOptions hardware_options;
7134 hardware_options.low_encode_usage_threshold_percent = 150;
7135 hardware_options.high_encode_usage_threshold_percent = 200;
7136 fake_encoder_.SetIsHardwareAccelerated(true);
7137
7138 video_source_.IncomingCapturedFrame(
7139 CreateFrame(2, kFrameWidth, kFrameHeight));
7140 WaitForEncodedFrame(2);
7141
7142 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7143 .low_encode_usage_threshold_percent,
7144 hardware_options.low_encode_usage_threshold_percent);
7145 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7146 .high_encode_usage_threshold_percent,
7147 hardware_options.high_encode_usage_threshold_percent);
7148
7149 video_stream_encoder_->Stop();
7150}
7151
Niels Möller6bb5ab92019-01-11 10:11:107152TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7153 const int kFrameWidth = 320;
7154 const int kFrameHeight = 240;
7155 const int kFps = 30;
7156 const int kTargetBitrateBps = 120000;
7157 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7158
Henrik Boström381d1092020-05-12 16:49:077159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077160 DataRate::BitsPerSec(kTargetBitrateBps),
7161 DataRate::BitsPerSec(kTargetBitrateBps),
7162 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107163
Tomas Gunnarsson612445e2020-09-21 12:31:237164 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:107165 max_framerate_ = kFps;
7166
7167 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7168 fake_encoder_.SimulateOvershoot(1.0);
7169 int num_dropped = 0;
7170 for (int i = 0; i < kNumFramesInRun; ++i) {
7171 video_source_.IncomingCapturedFrame(
7172 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7173 // Wait up to two frame durations for a frame to arrive.
7174 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7175 ++num_dropped;
7176 }
7177 timestamp_ms += 1000 / kFps;
7178 }
7179
Erik Språnga8d48ab2019-02-08 13:17:407180 // Framerate should be measured to be near the expected target rate.
7181 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7182
7183 // Frame drops should be within 5% of expected 0%.
7184 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:107185
7186 // Make encoder produce frames at double the expected bitrate during 3 seconds
7187 // of video, verify number of drops. Rate needs to be slightly changed in
7188 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 15:20:177189 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 15:44:427190 const RateControlSettings trials =
7191 RateControlSettings::ParseFromFieldTrials();
7192 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 15:20:177193 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 15:44:427194 // frame dropping since the adjuter will try to just lower the target
7195 // bitrate rather than drop frames. If network headroom can be used, it
7196 // doesn't push back as hard so we don't need quite as much overshoot.
7197 // These numbers are unfortunately a bit magical but there's not trivial
7198 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 10:10:097199 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 15:20:177200 }
7201 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 16:49:077202 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077203 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
7204 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
7205 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107206 num_dropped = 0;
7207 for (int i = 0; i < kNumFramesInRun; ++i) {
7208 video_source_.IncomingCapturedFrame(
7209 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7210 // Wait up to two frame durations for a frame to arrive.
7211 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7212 ++num_dropped;
7213 }
7214 timestamp_ms += 1000 / kFps;
7215 }
7216
Henrik Boström381d1092020-05-12 16:49:077217 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077218 DataRate::BitsPerSec(kTargetBitrateBps),
7219 DataRate::BitsPerSec(kTargetBitrateBps),
7220 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 13:17:407221
7222 // Target framerate should be still be near the expected target, despite
7223 // the frame drops.
7224 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7225
7226 // Frame drops should be within 5% of expected 50%.
7227 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:107228
7229 video_stream_encoder_->Stop();
7230}
7231
7232TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7233 const int kFrameWidth = 320;
7234 const int kFrameHeight = 240;
7235 const int kActualInputFps = 24;
7236 const int kTargetBitrateBps = 120000;
7237
7238 ASSERT_GT(max_framerate_, kActualInputFps);
7239
Tomas Gunnarsson612445e2020-09-21 12:31:237240 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:107241 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 16:49:077242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077243 DataRate::BitsPerSec(kTargetBitrateBps),
7244 DataRate::BitsPerSec(kTargetBitrateBps),
7245 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:107246
7247 // Insert 3 seconds of video, with an input fps lower than configured max.
7248 for (int i = 0; i < kActualInputFps * 3; ++i) {
7249 video_source_.IncomingCapturedFrame(
7250 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7251 // Wait up to two frame durations for a frame to arrive.
7252 WaitForEncodedFrame(timestamp_ms);
7253 timestamp_ms += 1000 / kActualInputFps;
7254 }
7255
7256 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7257
7258 video_stream_encoder_->Stop();
7259}
7260
Tomas Gunnarsson612445e2020-09-21 12:31:237261TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267262 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 16:49:077263 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077264 DataRate::BitsPerSec(kTargetBitrateBps),
7265 DataRate::BitsPerSec(kTargetBitrateBps),
7266 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:267267
7268 fake_encoder_.BlockNextEncode();
7269 video_source_.IncomingCapturedFrame(
7270 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7271 WaitForEncodedFrame(1);
7272 // On the very first frame full update should be forced.
7273 rect = fake_encoder_.GetLastUpdateRect();
7274 EXPECT_EQ(rect.offset_x, 0);
7275 EXPECT_EQ(rect.offset_y, 0);
7276 EXPECT_EQ(rect.height, codec_height_);
7277 EXPECT_EQ(rect.width, codec_width_);
7278 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7279 // call to ContinueEncode.
7280 video_source_.IncomingCapturedFrame(
7281 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7282 ExpectDroppedFrame();
7283 video_source_.IncomingCapturedFrame(
7284 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7285 ExpectDroppedFrame();
7286 fake_encoder_.ContinueEncode();
7287 WaitForEncodedFrame(3);
7288 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7289 rect = fake_encoder_.GetLastUpdateRect();
7290 EXPECT_EQ(rect.offset_x, 1);
7291 EXPECT_EQ(rect.offset_y, 0);
7292 EXPECT_EQ(rect.width, 10);
7293 EXPECT_EQ(rect.height, 1);
7294
7295 video_source_.IncomingCapturedFrame(
7296 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7297 WaitForEncodedFrame(4);
7298 // Previous frame was encoded, so no accumulation should happen.
7299 rect = fake_encoder_.GetLastUpdateRect();
7300 EXPECT_EQ(rect.offset_x, 0);
7301 EXPECT_EQ(rect.offset_y, 0);
7302 EXPECT_EQ(rect.width, 1);
7303 EXPECT_EQ(rect.height, 1);
7304
7305 video_stream_encoder_->Stop();
7306}
7307
Erik Språngd7329ca2019-02-21 20:19:537308TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 16:49:077309 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077310 DataRate::BitsPerSec(kTargetBitrateBps),
7311 DataRate::BitsPerSec(kTargetBitrateBps),
7312 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537313
7314 // First frame is always keyframe.
7315 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7316 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 14:43:587317 EXPECT_THAT(
7318 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127319 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537320
7321 // Insert delta frame.
7322 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7323 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 14:43:587324 EXPECT_THAT(
7325 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127326 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537327
7328 // Request next frame be a key-frame.
7329 video_stream_encoder_->SendKeyFrame();
7330 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7331 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 14:43:587332 EXPECT_THAT(
7333 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127334 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537335
7336 video_stream_encoder_->Stop();
7337}
7338
7339TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7340 // Setup simulcast with three streams.
7341 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077342 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077343 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7344 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7345 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537346 // Wait for all three layers before triggering event.
7347 sink_.SetNumExpectedLayers(3);
7348
7349 // First frame is always keyframe.
7350 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7351 WaitForEncodedFrame(1);
7352 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127353 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7354 VideoFrameType::kVideoFrameKey,
7355 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537356
7357 // Insert delta frame.
7358 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7359 WaitForEncodedFrame(2);
7360 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127361 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7362 VideoFrameType::kVideoFrameDelta,
7363 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537364
7365 // Request next frame be a key-frame.
7366 // Only first stream is configured to produce key-frame.
7367 video_stream_encoder_->SendKeyFrame();
7368 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7369 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 11:45:397370
7371 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7372 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 20:19:537373 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127374 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 11:45:397375 VideoFrameType::kVideoFrameKey,
7376 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537377
7378 video_stream_encoder_->Stop();
7379}
7380
7381TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7382 // Configure internal source factory and setup test again.
7383 encoder_factory_.SetHasInternalSource(true);
7384 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077385 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077386 DataRate::BitsPerSec(kTargetBitrateBps),
7387 DataRate::BitsPerSec(kTargetBitrateBps),
7388 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537389
7390 // Call encoder directly, simulating internal source where encoded frame
7391 // callback in VideoStreamEncoder is called despite no OnFrame().
7392 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7393 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:587394 EXPECT_THAT(
7395 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127396 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537397
Niels Möller8f7ce222019-03-21 14:43:587398 const std::vector<VideoFrameType> kDeltaFrame = {
7399 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:537400 // Need to set timestamp manually since manually for injected frame.
7401 VideoFrame frame = CreateFrame(101, nullptr);
7402 frame.set_timestamp(101);
7403 fake_encoder_.InjectFrame(frame, false);
7404 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:587405 EXPECT_THAT(
7406 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127407 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537408
7409 // Request key-frame. The forces a dummy frame down into the encoder.
7410 fake_encoder_.ExpectNullFrame();
7411 video_stream_encoder_->SendKeyFrame();
7412 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:587413 EXPECT_THAT(
7414 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127415 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537416
7417 video_stream_encoder_->Stop();
7418}
Erik Språngb7cb7b52019-02-26 14:52:337419
7420TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7421 // Configure internal source factory and setup test again.
7422 encoder_factory_.SetHasInternalSource(true);
7423 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077424 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077425 DataRate::BitsPerSec(kTargetBitrateBps),
7426 DataRate::BitsPerSec(kTargetBitrateBps),
7427 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 14:52:337428
7429 int64_t timestamp = 1;
7430 EncodedImage image;
Erik Språngb7cb7b52019-02-26 14:52:337431 image.capture_time_ms_ = ++timestamp;
7432 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7433 const int64_t kEncodeFinishDelayMs = 10;
7434 image.timing_.encode_start_ms = timestamp;
7435 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 09:12:577436 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 14:52:337437 // Wait for frame without incrementing clock.
7438 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7439 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7440 // capture timestamp should be kEncodeFinishDelayMs in the past.
7441 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 12:31:237442 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 14:52:337443
7444 video_stream_encoder_->Stop();
7445}
Mirta Dvornicic28f0eb22019-05-28 14:30:167446
7447TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237448 // SPS contains VUI with restrictions on the maximum number of reordered
7449 // pictures, there is no need to rewrite the bitstream to enable faster
7450 // decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167451 ResetEncoder("H264", 1, 1, 1, false);
7452
Mirta Dvornicic97910da2020-07-14 13:29:237453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7454 DataRate::BitsPerSec(kTargetBitrateBps),
7455 DataRate::BitsPerSec(kTargetBitrateBps),
7456 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7457 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167458
Mirta Dvornicic97910da2020-07-14 13:29:237459 fake_encoder_.SetEncodedImageData(
7460 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167461
Mirta Dvornicic97910da2020-07-14 13:29:237462 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7463 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167464
7465 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7466 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167467
7468 video_stream_encoder_->Stop();
7469}
7470
7471TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237472 // SPS does not contain VUI, the bitstream is will be rewritten with added
7473 // VUI with restrictions on the maximum number of reordered pictures to
7474 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167475 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7476 0x00, 0x00, 0x03, 0x03, 0xF4,
7477 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 14:30:167478 ResetEncoder("H264", 1, 1, 1, false);
7479
Mirta Dvornicic97910da2020-07-14 13:29:237480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7481 DataRate::BitsPerSec(kTargetBitrateBps),
7482 DataRate::BitsPerSec(kTargetBitrateBps),
7483 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7484 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167485
Mirta Dvornicic97910da2020-07-14 13:29:237486 fake_encoder_.SetEncodedImageData(
7487 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167488
Mirta Dvornicic97910da2020-07-14 13:29:237489 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7490 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167491
7492 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7493 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167494
7495 video_stream_encoder_->Stop();
7496}
7497
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147498TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7499 const int kFrameWidth = 1280;
7500 const int kFrameHeight = 720;
7501 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7502
Henrik Boström381d1092020-05-12 16:49:077503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077504 DataRate::BitsPerSec(kTargetBitrateBps),
7505 DataRate::BitsPerSec(kTargetBitrateBps),
7506 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147507 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7508
7509 // Insert a first video frame. It should be dropped because of downscale in
7510 // resolution.
Tomas Gunnarsson612445e2020-09-21 12:31:237511 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147512 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7513 frame.set_rotation(kVideoRotation_270);
7514 video_source_.IncomingCapturedFrame(frame);
7515
7516 ExpectDroppedFrame();
7517
7518 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237519 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147520 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7521 frame.set_rotation(kVideoRotation_90);
7522 video_source_.IncomingCapturedFrame(frame);
7523
7524 WaitForEncodedFrame(timestamp_ms);
7525 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7526
7527 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237528 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147529 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7530 frame.set_rotation(kVideoRotation_180);
7531 video_source_.IncomingCapturedFrame(frame);
7532
7533 WaitForEncodedFrame(timestamp_ms);
7534 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7535
7536 video_stream_encoder_->Stop();
7537}
7538
Erik Språng5056af02019-09-02 13:53:117539TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7540 const int kFrameWidth = 320;
7541 const int kFrameHeight = 180;
7542
7543 // Initial rate.
Henrik Boström381d1092020-05-12 16:49:077544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077545 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7546 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7547 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 13:53:117548 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327549 /*rtt_ms=*/0,
7550 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117551
7552 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237553 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 13:53:117554 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7555 frame.set_rotation(kVideoRotation_270);
7556 video_source_.IncomingCapturedFrame(frame);
7557 WaitForEncodedFrame(timestamp_ms);
7558
7559 // Set a target rate below the minimum allowed by the codec settings.
7560 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 17:46:077561 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7562 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 16:49:077563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 13:53:117564 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 11:36:557565 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 13:53:117566 /*link_allocation=*/target_rate,
7567 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327568 /*rtt_ms=*/0,
7569 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117570 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7571
7572 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7573 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7574 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 17:46:077575 DataRate allocation_sum =
7576 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 13:53:117577 EXPECT_EQ(min_rate, allocation_sum);
7578 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7579
7580 video_stream_encoder_->Stop();
7581}
7582
Rasmus Brandt5cad55b2019-12-19 08:47:117583TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 16:49:077584 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077585 DataRate::BitsPerSec(kTargetBitrateBps),
7586 DataRate::BitsPerSec(kTargetBitrateBps),
7587 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 10:50:237588 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 12:31:237589 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 10:50:237590 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7591 WaitForEncodedFrame(1);
7592
7593 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7594 ASSERT_TRUE(prev_rate_settings.has_value());
7595 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7596 kDefaultFramerate);
7597
7598 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7599 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7600 timestamp_ms += 1000 / kDefaultFramerate;
7601 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7602 WaitForEncodedFrame(timestamp_ms);
7603 }
7604 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7605 kDefaultFramerate);
7606 // Capture larger frame to trigger a reconfigure.
7607 codec_height_ *= 2;
7608 codec_width_ *= 2;
7609 timestamp_ms += 1000 / kDefaultFramerate;
7610 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7611 WaitForEncodedFrame(timestamp_ms);
7612
7613 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7614 auto current_rate_settings =
7615 fake_encoder_.GetAndResetLastRateControlSettings();
7616 // Ensure we have actually reconfigured twice
7617 // The rate settings should have been set again even though
7618 // they haven't changed.
7619 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 07:55:037620 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 10:50:237621
7622 video_stream_encoder_->Stop();
7623}
7624
philipeld9cc8c02019-09-16 12:53:407625struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 17:17:517626 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7627 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7628 MOCK_METHOD(void,
7629 RequestEncoderSwitch,
7630 (const webrtc::SdpVideoFormat& format),
7631 (override));
philipeld9cc8c02019-09-16 12:53:407632};
7633
philipel9b058032020-02-10 10:30:007634TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7635 constexpr int kDontCare = 100;
7636 StrictMock<MockEncoderSelector> encoder_selector;
7637 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7638 &fake_encoder_, &encoder_selector);
7639 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7640
7641 // Reset encoder for new configuration to take effect.
7642 ConfigureEncoder(video_encoder_config_.Copy());
7643
7644 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7645
7646 video_source_.IncomingCapturedFrame(
7647 CreateFrame(kDontCare, kDontCare, kDontCare));
7648 video_stream_encoder_->Stop();
7649
7650 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7651 // to it's factory, so in order for the encoder instance in the
7652 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7653 // reset the |video_stream_encoder_| here.
7654 video_stream_encoder_.reset();
7655}
7656
7657TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7658 constexpr int kDontCare = 100;
7659
7660 NiceMock<MockEncoderSelector> encoder_selector;
7661 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7662 video_send_config_.encoder_settings.encoder_switch_request_callback =
7663 &switch_callback;
7664 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7665 &fake_encoder_, &encoder_selector);
7666 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7667
7668 // Reset encoder for new configuration to take effect.
7669 ConfigureEncoder(video_encoder_config_.Copy());
7670
Mirta Dvornicic4f34d782020-02-26 12:01:197671 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 10:30:007672 .WillByDefault(Return(SdpVideoFormat("AV1")));
7673 EXPECT_CALL(switch_callback,
7674 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7675 Field(&SdpVideoFormat::name, "AV1"))));
7676
Henrik Boström381d1092020-05-12 16:49:077677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077678 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7679 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7680 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 10:30:007681 /*fraction_lost=*/0,
7682 /*rtt_ms=*/0,
7683 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 13:56:427684 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 10:30:007685
7686 video_stream_encoder_->Stop();
7687}
7688
7689TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7690 constexpr int kSufficientBitrateToNotDrop = 1000;
7691 constexpr int kDontCare = 100;
7692
7693 NiceMock<MockVideoEncoder> video_encoder;
7694 NiceMock<MockEncoderSelector> encoder_selector;
7695 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7696 video_send_config_.encoder_settings.encoder_switch_request_callback =
7697 &switch_callback;
7698 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7699 &video_encoder, &encoder_selector);
7700 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7701
7702 // Reset encoder for new configuration to take effect.
7703 ConfigureEncoder(video_encoder_config_.Copy());
7704
7705 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7706 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7707 // not fail.
Henrik Boström381d1092020-05-12 16:49:077708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077709 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7710 /*stable_target_bitrate=*/
7711 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7712 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 10:30:007713 /*fraction_lost=*/0,
7714 /*rtt_ms=*/0,
7715 /*cwnd_reduce_ratio=*/0);
7716
7717 ON_CALL(video_encoder, Encode(_, _))
7718 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7719 ON_CALL(encoder_selector, OnEncoderBroken())
7720 .WillByDefault(Return(SdpVideoFormat("AV2")));
7721
7722 rtc::Event encode_attempted;
7723 EXPECT_CALL(switch_callback,
7724 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7725 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7726 EXPECT_EQ(format.name, "AV2");
7727 encode_attempted.Set();
7728 });
7729
7730 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7731 encode_attempted.Wait(3000);
7732
Tomas Gunnarssond41c2a62020-09-21 13:56:427733 AdvanceTime(TimeDelta::Millis(0));
7734
philipel9b058032020-02-10 10:30:007735 video_stream_encoder_->Stop();
7736
7737 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7738 // to it's factory, so in order for the encoder instance in the
7739 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7740 // reset the |video_stream_encoder_| here.
7741 video_stream_encoder_.reset();
7742}
7743
Evan Shrubsole7c079f62019-09-26 07:55:037744TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117745 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037746 const int kFrameWidth = 320;
7747 const int kFrameHeight = 180;
7748
7749 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077750 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037752 /*target_bitrate=*/rate,
7753 /*stable_target_bitrate=*/rate,
7754 /*link_allocation=*/rate,
7755 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327756 /*rtt_ms=*/0,
7757 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037758
7759 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237760 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037761 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7762 frame.set_rotation(kVideoRotation_270);
7763 video_source_.IncomingCapturedFrame(frame);
7764 WaitForEncodedFrame(timestamp_ms);
7765 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7766
7767 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 17:46:077768 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037770 /*target_bitrate=*/new_stable_rate,
7771 /*stable_target_bitrate=*/new_stable_rate,
7772 /*link_allocation=*/rate,
7773 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327774 /*rtt_ms=*/0,
7775 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037776 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7777 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7778 video_stream_encoder_->Stop();
7779}
7780
7781TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117782 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037783 const int kFrameWidth = 320;
7784 const int kFrameHeight = 180;
7785
7786 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077787 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037789 /*target_bitrate=*/rate,
7790 /*stable_target_bitrate=*/rate,
7791 /*link_allocation=*/rate,
7792 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327793 /*rtt_ms=*/0,
7794 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037795
7796 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237797 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037798 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7799 frame.set_rotation(kVideoRotation_270);
7800 video_source_.IncomingCapturedFrame(frame);
7801 WaitForEncodedFrame(timestamp_ms);
7802 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7803
7804 // Set a higher target rate without changing the link_allocation. Should not
7805 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077806 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077807 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037808 /*target_bitrate=*/rate,
7809 /*stable_target_bitrate=*/new_stable_rate,
7810 /*link_allocation=*/rate,
7811 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327812 /*rtt_ms=*/0,
7813 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037814 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7815 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7816 video_stream_encoder_->Stop();
7817}
7818
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177819TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7820 test::ScopedFieldTrials field_trials(
7821 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7822 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7823 const int kFramerateFps = 30;
7824 const int kWidth = 1920;
7825 const int kHeight = 1080;
7826 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7827 // Works on screenshare mode.
7828 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7829 // We rely on the automatic resolution adaptation, but we handle framerate
7830 // adaptation manually by mocking the stats proxy.
7831 video_source_.set_adaptation_enabled(true);
7832
7833 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 16:49:077834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077835 DataRate::BitsPerSec(kTargetBitrateBps),
7836 DataRate::BitsPerSec(kTargetBitrateBps),
7837 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177838 video_stream_encoder_->SetSource(&video_source_,
7839 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:397840 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177841
7842 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7843 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7844
7845 // Pass enough frames with the full update to trigger animation detection.
7846 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237847 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177848 frame.set_ntp_time_ms(timestamp_ms);
7849 frame.set_timestamp_us(timestamp_ms * 1000);
7850 video_source_.IncomingCapturedFrame(frame);
7851 WaitForEncodedFrame(timestamp_ms);
7852 }
7853
7854 // Resolution should be limited.
7855 rtc::VideoSinkWants expected;
7856 expected.max_framerate_fps = kFramerateFps;
7857 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 14:19:547858 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177859
7860 // Pass one frame with no known update.
7861 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 12:31:237862 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177863 frame.set_ntp_time_ms(timestamp_ms);
7864 frame.set_timestamp_us(timestamp_ms * 1000);
7865 frame.clear_update_rect();
7866
7867 video_source_.IncomingCapturedFrame(frame);
7868 WaitForEncodedFrame(timestamp_ms);
7869
7870 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 12:08:397871 EXPECT_THAT(video_source_.sink_wants(),
7872 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177873
7874 video_stream_encoder_->Stop();
7875}
7876
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327877TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7878 const int kWidth = 720; // 540p adapted down.
7879 const int kHeight = 405;
7880 const int kNumFrames = 3;
7881 // Works on screenshare mode.
7882 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7883 /*num_spatial_layers=*/2, /*screenshare=*/true);
7884
7885 video_source_.set_adaptation_enabled(true);
7886
7887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7888 DataRate::BitsPerSec(kTargetBitrateBps),
7889 DataRate::BitsPerSec(kTargetBitrateBps),
7890 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7891
7892 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7893
7894 // Pass enough frames with the full update to trigger animation detection.
7895 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237896 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327897 frame.set_ntp_time_ms(timestamp_ms);
7898 frame.set_timestamp_us(timestamp_ms * 1000);
7899 video_source_.IncomingCapturedFrame(frame);
7900 WaitForEncodedFrame(timestamp_ms);
7901 }
7902
7903 video_stream_encoder_->Stop();
7904}
7905
Yun Zhang1e4d4fd2020-09-30 07:22:467906TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7907 const float downscale_factors[] = {4.0, 2.0, 1.0};
7908 const int number_layers =
7909 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7910 VideoEncoderConfig config;
7911 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7912 for (int i = 0; i < number_layers; ++i) {
7913 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7914 config.simulcast_layers[i].active = true;
7915 }
7916 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:437917 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 07:22:467918 "VP8", /*max qp*/ 56, /*screencast*/ false,
7919 /*screenshare enabled*/ false);
7920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7921 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7922 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7923 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7924
7925 // First initialization.
7926 // Encoder should be initialized. Next frame should be key frame.
7927 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7928 sink_.SetNumExpectedLayers(number_layers);
7929 int64_t timestamp_ms = kFrameIntervalMs;
7930 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7931 WaitForEncodedFrame(timestamp_ms);
7932 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7933 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7934 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7935 VideoFrameType::kVideoFrameKey,
7936 VideoFrameType::kVideoFrameKey}));
7937
7938 // Disable top layer.
7939 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7940 config.simulcast_layers[number_layers - 1].active = false;
7941 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7942 sink_.SetNumExpectedLayers(number_layers - 1);
7943 timestamp_ms += kFrameIntervalMs;
7944 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7945 WaitForEncodedFrame(timestamp_ms);
7946 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7947 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7948 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7949 VideoFrameType::kVideoFrameDelta,
7950 VideoFrameType::kVideoFrameDelta}));
7951
7952 // Re-enable top layer.
7953 // Encoder should be re-initialized. Next frame should be key frame.
7954 config.simulcast_layers[number_layers - 1].active = true;
7955 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7956 sink_.SetNumExpectedLayers(number_layers);
7957 timestamp_ms += kFrameIntervalMs;
7958 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7959 WaitForEncodedFrame(timestamp_ms);
7960 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7961 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7962 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7963 VideoFrameType::kVideoFrameKey,
7964 VideoFrameType::kVideoFrameKey}));
7965
7966 // Top layer max rate change.
7967 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7968 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7969 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7970 sink_.SetNumExpectedLayers(number_layers);
7971 timestamp_ms += kFrameIntervalMs;
7972 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7973 WaitForEncodedFrame(timestamp_ms);
7974 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7975 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7976 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7977 VideoFrameType::kVideoFrameDelta,
7978 VideoFrameType::kVideoFrameDelta}));
7979
7980 // Top layer resolution change.
7981 // Encoder should be re-initialized. Next frame should be key frame.
7982 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7983 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7984 sink_.SetNumExpectedLayers(number_layers);
7985 timestamp_ms += kFrameIntervalMs;
7986 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7987 WaitForEncodedFrame(timestamp_ms);
7988 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7989 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7990 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7991 VideoFrameType::kVideoFrameKey,
7992 VideoFrameType::kVideoFrameKey}));
7993 video_stream_encoder_->Stop();
7994}
7995
Henrik Boström1124ed12021-02-25 09:30:397996TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7997 const int kFrameWidth = 1280;
7998 const int kFrameHeight = 720;
7999
8000 SetUp();
8001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8002 DataRate::BitsPerSec(kTargetBitrateBps),
8003 DataRate::BitsPerSec(kTargetBitrateBps),
8004 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8005
8006 // Capturing a frame should reconfigure the encoder and expose the encoder
8007 // resolution, which is the same as the input frame.
8008 int64_t timestamp_ms = kFrameIntervalMs;
8009 video_source_.IncomingCapturedFrame(
8010 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8011 WaitForEncodedFrame(timestamp_ms);
8012 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8013 EXPECT_THAT(video_source_.sink_wants().resolutions,
8014 ::testing::ElementsAreArray(
8015 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8016
8017 video_stream_encoder_->Stop();
8018}
8019
8020TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8021 // Pick downscale factors such that we never encode at full resolution - this
8022 // is an interesting use case. The frame resolution influences the encoder
8023 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
8024 // encoder should not ask for the frame resolution. This allows video frames
8025 // to have the appearence of one resolution but optimize its internal buffers
8026 // for what is actually encoded.
8027 const size_t kNumSimulcastLayers = 3u;
8028 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8029 const int kFrameWidth = 1280;
8030 const int kFrameHeight = 720;
8031 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8032 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8033 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8034 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8035 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8036 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8037
8038 VideoEncoderConfig config;
8039 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8040 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8041 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8042 config.simulcast_layers[i].active = true;
8043 }
8044 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 17:21:438045 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 09:30:398046 "VP8", /*max qp*/ 56, /*screencast*/ false,
8047 /*screenshare enabled*/ false);
8048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8049 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8050 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8051 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
8052
8053 // Capture a frame with all layers active.
8054 int64_t timestamp_ms = kFrameIntervalMs;
8055 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8056 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8057 video_source_.IncomingCapturedFrame(
8058 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8059 WaitForEncodedFrame(timestamp_ms);
8060 // Expect encoded resolutions to match the expected simulcast layers.
8061 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8062 EXPECT_THAT(
8063 video_source_.sink_wants().resolutions,
8064 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8065
8066 // Capture a frame with one of the layers inactive.
8067 timestamp_ms += kFrameIntervalMs;
8068 config.simulcast_layers[2].active = false;
8069 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8070 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8071 video_source_.IncomingCapturedFrame(
8072 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8073 WaitForEncodedFrame(timestamp_ms);
8074
8075 // Expect encoded resolutions to match the expected simulcast layers.
8076 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8077 EXPECT_THAT(video_source_.sink_wants().resolutions,
8078 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8079
8080 // Capture a frame with all but one layer turned off.
8081 timestamp_ms += kFrameIntervalMs;
8082 config.simulcast_layers[1].active = false;
8083 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8084 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8085 video_source_.IncomingCapturedFrame(
8086 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8087 WaitForEncodedFrame(timestamp_ms);
8088
8089 // Expect encoded resolutions to match the expected simulcast layers.
8090 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8091 EXPECT_THAT(video_source_.sink_wants().resolutions,
8092 ::testing::ElementsAreArray({kLayer0Size}));
8093
8094 video_stream_encoder_->Stop();
8095}
8096
Sergey Silkin0e42cf72021-03-15 09:12:578097TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 09:12:578098 ResetEncoder("VP8", 1, 1, 1, false);
8099
Niels Möller8b692902021-06-14 10:04:578100 // Force encoder reconfig.
8101 video_source_.IncomingCapturedFrame(
8102 CreateFrame(1, codec_width_, codec_height_));
8103 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8104
Sergey Silkin0e42cf72021-03-15 09:12:578105 // Set QP on encoded frame and pass the frame to encode complete callback.
8106 // Since QP is present QP parsing won't be triggered and the original value
8107 // should be kept.
8108 EncodedImage encoded_image;
8109 encoded_image.qp_ = 123;
8110 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8111 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8112 CodecSpecificInfo codec_info;
8113 codec_info.codecType = kVideoCodecVP8;
8114 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8115 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8116 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8117 video_stream_encoder_->Stop();
8118}
8119
8120TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 09:12:578121 ResetEncoder("VP8", 1, 1, 1, false);
8122
Niels Möller8b692902021-06-14 10:04:578123 // Force encoder reconfig.
8124 video_source_.IncomingCapturedFrame(
8125 CreateFrame(1, codec_width_, codec_height_));
8126 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8127
Sergey Silkin0e42cf72021-03-15 09:12:578128 // Pass an encoded frame without QP to encode complete callback. QP should be
8129 // parsed and set.
8130 EncodedImage encoded_image;
8131 encoded_image.qp_ = -1;
8132 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8133 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8134 CodecSpecificInfo codec_info;
8135 codec_info.codecType = kVideoCodecVP8;
8136 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8137 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8138 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8139 video_stream_encoder_->Stop();
8140}
8141
8142TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
8143 webrtc::test::ScopedFieldTrials field_trials(
8144 "WebRTC-QpParsingKillSwitch/Enabled/");
8145
Sergey Silkin0e42cf72021-03-15 09:12:578146 ResetEncoder("VP8", 1, 1, 1, false);
8147
Niels Möller8b692902021-06-14 10:04:578148 // Force encoder reconfig.
8149 video_source_.IncomingCapturedFrame(
8150 CreateFrame(1, codec_width_, codec_height_));
8151 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8152
Sergey Silkin0e42cf72021-03-15 09:12:578153 EncodedImage encoded_image;
8154 encoded_image.qp_ = -1;
8155 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8156 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8157 CodecSpecificInfo codec_info;
8158 codec_info.codecType = kVideoCodecVP8;
8159 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8160 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8161 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8162 video_stream_encoder_->Stop();
8163}
8164
Sergey Silkind19e3b92021-03-16 10:05:308165TEST_F(VideoStreamEncoderTest,
8166 QualityScalingNotAllowed_QualityScalingDisabled) {
8167 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8168
8169 // Disable scaling settings in encoder info.
8170 fake_encoder_.SetQualityScaling(false);
8171 // Disable quality scaling in encoder config.
8172 video_encoder_config.is_quality_scaling_allowed = false;
8173 ConfigureEncoder(std::move(video_encoder_config));
8174
8175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8176 DataRate::BitsPerSec(kTargetBitrateBps),
8177 DataRate::BitsPerSec(kTargetBitrateBps),
8178 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8179
8180 test::FrameForwarder source;
8181 video_stream_encoder_->SetSource(
8182 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8183 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8184 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8185
8186 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8187 WaitForEncodedFrame(1);
8188 video_stream_encoder_->TriggerQualityLow();
8189 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8190
8191 video_stream_encoder_->Stop();
8192}
8193
8194#if !defined(WEBRTC_IOS)
8195// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8196// disabled by default on iOS.
8197TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8198 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8199
8200 // Disable scaling settings in encoder info.
8201 fake_encoder_.SetQualityScaling(false);
8202 // Enable quality scaling in encoder config.
8203 video_encoder_config.is_quality_scaling_allowed = true;
8204 ConfigureEncoder(std::move(video_encoder_config));
8205
8206 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8207 DataRate::BitsPerSec(kTargetBitrateBps),
8208 DataRate::BitsPerSec(kTargetBitrateBps),
8209 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8210
8211 test::FrameForwarder source;
8212 video_stream_encoder_->SetSource(
8213 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8214 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8215 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8216
8217 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8218 WaitForEncodedFrame(1);
8219 video_stream_encoder_->TriggerQualityLow();
8220 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8221
8222 video_stream_encoder_->Stop();
8223}
8224#endif
8225
Henrik Boström56db9ff2021-03-24 08:06:458226// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8227class VideoStreamEncoderWithRealEncoderTest
8228 : public VideoStreamEncoderTest,
8229 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8230 public:
8231 VideoStreamEncoderWithRealEncoderTest()
8232 : VideoStreamEncoderTest(),
8233 codec_type_(std::get<0>(GetParam())),
8234 allow_i420_conversion_(std::get<1>(GetParam())) {}
8235
8236 void SetUp() override {
8237 VideoStreamEncoderTest::SetUp();
8238 std::unique_ptr<VideoEncoder> encoder;
8239 switch (codec_type_) {
8240 case kVideoCodecVP8:
8241 encoder = VP8Encoder::Create();
8242 break;
8243 case kVideoCodecVP9:
8244 encoder = VP9Encoder::Create();
8245 break;
8246 case kVideoCodecAV1:
8247 encoder = CreateLibaomAv1Encoder();
8248 break;
8249 case kVideoCodecH264:
8250 encoder =
8251 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8252 break;
8253 case kVideoCodecMultiplex:
8254 mock_encoder_factory_for_multiplex_ =
8255 std::make_unique<MockVideoEncoderFactory>();
8256 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8257 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8258 .WillRepeatedly([] { return VP8Encoder::Create(); });
8259 encoder = std::make_unique<MultiplexEncoderAdapter>(
8260 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8261 false);
8262 break;
8263 default:
8264 RTC_NOTREACHED();
8265 }
8266 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8267 }
8268
8269 void TearDown() override {
8270 video_stream_encoder_->Stop();
8271 // Ensure |video_stream_encoder_| is destroyed before
8272 // |encoder_proxy_factory_|.
8273 video_stream_encoder_.reset();
8274 VideoStreamEncoderTest::TearDown();
8275 }
8276
8277 protected:
8278 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8279 std::unique_ptr<VideoEncoder> encoder) {
8280 // Configure VSE to use the encoder.
8281 encoder_ = std::move(encoder);
8282 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8283 encoder_.get(), &encoder_selector_);
8284 video_send_config_.encoder_settings.encoder_factory =
8285 encoder_proxy_factory_.get();
8286 VideoEncoderConfig video_encoder_config;
8287 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8288 video_encoder_config_ = video_encoder_config.Copy();
8289 ConfigureEncoder(video_encoder_config_.Copy());
8290
8291 // Set bitrate to ensure frame is not dropped.
8292 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8293 DataRate::BitsPerSec(kTargetBitrateBps),
8294 DataRate::BitsPerSec(kTargetBitrateBps),
8295 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8296 }
8297
8298 const VideoCodecType codec_type_;
8299 const bool allow_i420_conversion_;
8300 NiceMock<MockEncoderSelector> encoder_selector_;
8301 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8302 std::unique_ptr<VideoEncoder> encoder_;
8303 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8304};
8305
8306TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8307 auto native_i420_frame = test::CreateMappableNativeFrame(
8308 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8309 video_source_.IncomingCapturedFrame(native_i420_frame);
8310 WaitForEncodedFrame(codec_width_, codec_height_);
8311
8312 auto mappable_native_buffer =
8313 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8314 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8315 mappable_native_buffer->GetMappedFramedBuffers();
8316 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8317 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8318 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8319 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8320}
8321
8322TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8323 auto native_nv12_frame = test::CreateMappableNativeFrame(
8324 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8325 video_source_.IncomingCapturedFrame(native_nv12_frame);
8326 WaitForEncodedFrame(codec_width_, codec_height_);
8327
8328 auto mappable_native_buffer =
8329 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8330 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8331 mappable_native_buffer->GetMappedFramedBuffers();
8332 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8333 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8334 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8335 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8336
8337 if (!allow_i420_conversion_) {
8338 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8339 }
8340}
8341
Erik Språng7444b192021-06-02 12:02:138342TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8343 if (codec_type_ == kVideoCodecMultiplex) {
8344 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8345 return;
8346 }
8347
8348 const size_t kNumSpatialLayers = 3u;
8349 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8350 const int kFrameWidth = 1280;
8351 const int kFrameHeight = 720;
8352 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8353 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8354 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8355 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8356 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8357 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8358
8359 VideoEncoderConfig config;
8360 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8361 test::FillEncoderConfiguration(codec_type_, 1, &config);
8362 config.max_bitrate_bps = kSimulcastTargetBitrateBps;
8363 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8364 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8365 vp9_settings.numberOfTemporalLayers = 3;
8366 vp9_settings.automaticResizeOn = false;
8367 config.encoder_specific_settings =
8368 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8369 vp9_settings);
8370 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8371 /*fps=*/30.0,
8372 /*first_active_layer=*/0,
8373 /*num_spatial_layers=*/3,
8374 /*num_temporal_layers=*/3,
8375 /*is_screenshare=*/false);
8376 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8377 test::FillEncoderConfiguration(codec_type_, 1, &config);
8378 config.max_bitrate_bps = kSimulcastTargetBitrateBps;
8379 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8380 /*fps=*/30.0,
8381 /*first_active_layer=*/0,
8382 /*num_spatial_layers=*/3,
8383 /*num_temporal_layers=*/3,
8384 /*is_screenshare=*/false);
8385 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8386 } else {
8387 // Simulcast for VP8/H264.
8388 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8389 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8390 config.simulcast_layers[i].scale_resolution_down_by =
8391 kDownscaleFactors[i];
8392 config.simulcast_layers[i].active = true;
8393 }
8394 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8395 // Turn off frame dropping to prevent flakiness.
8396 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8397 h264_settings.frameDroppingOn = false;
8398 config.encoder_specific_settings = rtc::make_ref_counted<
8399 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8400 }
8401 }
8402
8403 auto set_layer_active = [&](int layer_idx, bool active) {
8404 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8405 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8406 config.spatial_layers[layer_idx].active = active;
8407 } else {
8408 config.simulcast_layers[layer_idx].active = active;
8409 }
8410 };
8411
8412 config.video_stream_factory =
8413 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8414 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8415 /*screencast*/ false,
8416 /*screenshare enabled*/ false);
8417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8418 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8419 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8420 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
8421
8422 // Capture a frame with all layers active.
8423 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8424 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8425 int64_t timestamp_ms = kFrameIntervalMs;
8426 video_source_.IncomingCapturedFrame(
8427 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8428
8429 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8430 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8431
8432 // Capture a frame with one of the layers inactive.
8433 set_layer_active(2, false);
8434 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8435 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8436 timestamp_ms += kFrameIntervalMs;
8437 video_source_.IncomingCapturedFrame(
8438 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8439 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8440
8441 // New target bitrates signaled based on lower resolution.
8442 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8444 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8445 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8446
8447 // Re-enable the top layer.
8448 set_layer_active(2, true);
8449 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8450 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8451 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8452
8453 // Bitrate target adjusted back up to enable HD layer...
8454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8455 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8456 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8457 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8458
8459 // ...then add a new frame.
8460 timestamp_ms += kFrameIntervalMs;
8461 video_source_.IncomingCapturedFrame(
8462 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8463 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8464 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8465
8466 video_stream_encoder_->Stop();
8467}
8468
Henrik Boström56db9ff2021-03-24 08:06:458469std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8470 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8471 VideoCodecType codec_type = std::get<0>(info.param);
8472 bool allow_i420_conversion = std::get<1>(info.param);
8473 std::string str;
8474 switch (codec_type) {
8475 case kVideoCodecGeneric:
8476 str = "Generic";
8477 break;
8478 case kVideoCodecVP8:
8479 str = "VP8";
8480 break;
8481 case kVideoCodecVP9:
8482 str = "VP9";
8483 break;
8484 case kVideoCodecAV1:
8485 str = "AV1";
8486 break;
8487 case kVideoCodecH264:
8488 str = "H264";
8489 break;
8490 case kVideoCodecMultiplex:
8491 str = "Multiplex";
8492 break;
8493 default:
8494 RTC_NOTREACHED();
8495 }
8496 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8497 return str;
8498}
8499
8500constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8501 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8502constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8503 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8504constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8505 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8506constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8507 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8508#if defined(WEBRTC_USE_H264)
8509constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8510 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8511
8512// The windows compiler does not tolerate #if statements inside the
8513// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8514// and without H264).
8515INSTANTIATE_TEST_SUITE_P(
8516 All,
8517 VideoStreamEncoderWithRealEncoderTest,
8518 ::testing::Values(kVP8DisallowConversion,
8519 kVP9DisallowConversion,
8520 kAV1AllowConversion,
8521 kMultiplexDisallowConversion,
8522 kH264AllowConversion),
8523 TestParametersVideoCodecAndAllowI420ConversionToString);
8524#else
8525INSTANTIATE_TEST_SUITE_P(
8526 All,
8527 VideoStreamEncoderWithRealEncoderTest,
8528 ::testing::Values(kVP8DisallowConversion,
8529 kVP9DisallowConversion,
8530 kAV1AllowConversion,
8531 kMultiplexDisallowConversion),
8532 TestParametersVideoCodecAndAllowI420ConversionToString);
8533#endif
8534
perkj26091b12016-09-01 08:17:408535} // namespace webrtc