blob: e665d809e3f533e7b53ba2b133858cdd1c2137f8 [file] [log] [blame]
perkj26091b12016-09-01 08:17:401/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 08:30:3111#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 15:24:5913#include <algorithm>
perkj803d97f2016-11-01 18:45:4614#include <limits>
Danil Chapovalovd3ba2362019-04-10 15:01:2315#include <memory>
Per512ecb32016-09-23 13:52:0616#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 14:30:1618#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 15:01:2319#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 09:20:0920#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 10:30:0021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 18:02:5622#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3123#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 07:15:1324#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 09:36:5525#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 16:10:5726#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 12:57:5727#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 08:30:3128#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 10:56:2029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 11:02:3630#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 15:26:0131#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 14:30:1632#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 13:59:1233#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 17:11:0034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 15:50:0035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 17:32:3736#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 10:24:3337#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 11:06:5938#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2339#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 09:39:5140#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 14:29:0941#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3142#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 17:11:0043#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 11:13:3244#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 15:20:1745#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 06:51:1046#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3147#include "test/encoder_settings.h"
48#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 08:51:4049#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 14:59:0050#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3151#include "test/gmock.h"
52#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 12:31:2353#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 07:07:2454#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3155#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 08:17:4056
57namespace webrtc {
58
sprang57c2fff2017-01-16 14:24:0259using ::testing::_;
philipeld9cc8c02019-09-16 12:53:4060using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 09:28:4161using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 12:08:3962using ::testing::Eq;
philipeld9cc8c02019-09-16 12:53:4063using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 12:08:3964using ::testing::Ge;
65using ::testing::Gt;
66using ::testing::Le;
67using ::testing::Lt;
philipel9b058032020-02-10 10:30:0068using ::testing::Matcher;
69using ::testing::NiceMock;
70using ::testing::Return;
Per Kjellander4190ce92020-12-15 16:24:5571using ::testing::SizeIs;
philipeld9cc8c02019-09-16 12:53:4072using ::testing::StrictMock;
kthelgason876222f2016-11-29 09:44:1173
perkj803d97f2016-11-01 18:45:4674namespace {
Åsa Persson8c1bf952018-09-13 08:42:1975const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 14:56:0076const int kQpLow = 1;
77const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 08:42:1978const int kMinFramerateFps = 2;
79const int kMinBalancedFramerateFps = 7;
80const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 12:37:0081const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 20:19:5382const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 12:18:3483const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 20:19:5384const uint32_t kSimulcastTargetBitrateBps = 3150000;
85const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 15:02:2286const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 11:21:0787const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 08:42:1988const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 08:48:4889const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 11:12:1990const VideoEncoder::ResolutionBitrateLimits
91 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
92const VideoEncoder::ResolutionBitrateLimits
93 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 12:37:0094
Mirta Dvornicic28f0eb22019-05-28 14:30:1695uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
96 0x00, 0x00, 0x03, 0x03, 0xF4,
97 0x05, 0x03, 0xC7, 0xE0, 0x1B,
98 0x41, 0x10, 0x8D, 0x00};
99
Sergey Silkin0e42cf72021-03-15 09:12:57100const uint8_t kCodedFrameVp8Qp25[] = {
101 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
102 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
103 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
104
perkj803d97f2016-11-01 18:45:46105class TestBuffer : public webrtc::I420Buffer {
106 public:
107 TestBuffer(rtc::Event* event, int width, int height)
108 : I420Buffer(width, height), event_(event) {}
109
110 private:
111 friend class rtc::RefCountedObject<TestBuffer>;
112 ~TestBuffer() override {
113 if (event_)
114 event_->Set();
115 }
116 rtc::Event* const event_;
117};
118
Noah Richards51db4212019-06-12 13:59:12119// A fake native buffer that can't be converted to I420.
120class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
121 public:
122 FakeNativeBuffer(rtc::Event* event, int width, int height)
123 : event_(event), width_(width), height_(height) {}
124 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
125 int width() const override { return width_; }
126 int height() const override { return height_; }
127 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
128 return nullptr;
129 }
130
131 private:
132 friend class rtc::RefCountedObject<FakeNativeBuffer>;
133 ~FakeNativeBuffer() override {
134 if (event_)
135 event_->Set();
136 }
137 rtc::Event* const event_;
138 const int width_;
139 const int height_;
140};
141
Evan Shrubsole895556e2020-10-05 07:15:13142// A fake native buffer that is backed by an NV12 buffer.
143class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
144 public:
145 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
146 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
147
148 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
149 int width() const override { return nv12_buffer_->width(); }
150 int height() const override { return nv12_buffer_->height(); }
151 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
152 return nv12_buffer_->ToI420();
153 }
Evan Shrubsoleb556b082020-10-08 12:56:45154 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
155 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
156 if (absl::c_find(types, Type::kNV12) != types.end()) {
157 return nv12_buffer_;
158 }
159 return nullptr;
160 }
Evan Shrubsole895556e2020-10-05 07:15:13161 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
162
163 private:
164 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
165 ~FakeNV12NativeBuffer() override {
166 if (event_)
167 event_->Set();
168 }
169 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
170 rtc::Event* const event_;
171};
172
Niels Möller7dc26b72017-12-06 09:27:48173class CpuOveruseDetectorProxy : public OveruseFrameDetector {
174 public:
Niels Möllerd1f7eb62018-03-28 14:40:58175 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
176 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 16:49:07177 last_target_framerate_fps_(-1),
178 framerate_updated_event_(true /* manual_reset */,
179 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 09:27:48180 virtual ~CpuOveruseDetectorProxy() {}
181
182 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 11:13:32183 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48184 last_target_framerate_fps_ = framerate_fps;
185 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 16:49:07186 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 09:27:48187 }
188
189 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 11:13:32190 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 09:27:48191 return last_target_framerate_fps_;
192 }
193
Niels Möller4db138e2018-04-19 07:04:13194 CpuOveruseOptions GetOptions() { return options_; }
195
Henrik Boström381d1092020-05-12 16:49:07196 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
197
Niels Möller7dc26b72017-12-06 09:27:48198 private:
Markus Handella3765182020-07-08 11:13:32199 Mutex lock_;
Niels Möller7dc26b72017-12-06 09:27:48200 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 16:49:07201 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 09:27:48202};
203
Henrik Boström0f0aa9c2020-06-02 11:02:36204class FakeVideoSourceRestrictionsListener
205 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 16:49:07206 public:
Henrik Boström0f0aa9c2020-06-02 11:02:36207 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 16:49:07208 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 11:02:36209 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 16:49:07210 RTC_DCHECK(was_restrictions_updated_);
211 }
212
213 rtc::Event* restrictions_updated_event() {
214 return &restrictions_updated_event_;
215 }
216
Henrik Boström0f0aa9c2020-06-02 11:02:36217 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 16:49:07218 void OnVideoSourceRestrictionsUpdated(
219 VideoSourceRestrictions restrictions,
220 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 09:47:46221 rtc::scoped_refptr<Resource> reason,
222 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 16:49:07223 was_restrictions_updated_ = true;
224 restrictions_updated_event_.Set();
225 }
226
227 private:
228 bool was_restrictions_updated_;
229 rtc::Event restrictions_updated_event_;
230};
231
Evan Shrubsole5fd40602020-05-25 14:19:54232auto WantsFps(Matcher<int> fps_matcher) {
233 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
234 fps_matcher);
235}
236
237auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
238 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
239 AllOf(max_pixel_matcher, Gt(0)));
240}
241
Evan Shrubsole5cd7eb82020-05-25 12:08:39242auto ResolutionMax() {
243 return AllOf(
Evan Shrubsole5fd40602020-05-25 14:19:54244 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 12:08:39245 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
246 Eq(absl::nullopt)));
247}
248
249auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 14:19:54250 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 12:08:39251}
252
253auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 14:19:54254 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 12:08:39255}
256
257auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54258 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 12:08:39259}
260
261auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 14:19:54262 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 12:08:39263}
264
265auto FpsMaxResolutionMax() {
266 return AllOf(FpsMax(), ResolutionMax());
267}
268
269auto UnlimitedSinkWants() {
270 return AllOf(FpsUnlimited(), ResolutionMax());
271}
272
273auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
274 Matcher<int> fps_range_matcher;
275
276 if (last_frame_pixels <= 320 * 240) {
277 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 15:50:00278 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 12:08:39279 fps_range_matcher = AllOf(Ge(10), Le(15));
280 } else if (last_frame_pixels <= 640 * 480) {
281 fps_range_matcher = Ge(15);
282 } else {
283 fps_range_matcher = Eq(kDefaultFramerate);
284 }
285 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
286 fps_range_matcher);
287}
288
Evan Shrubsole5fd40602020-05-25 14:19:54289auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
290 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
291 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
292}
293
294auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
295 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
296}
297
298auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
299 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
300}
301
302auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
303 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
304 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
305}
306
307auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
308 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
309 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
310}
311
312auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
313 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
314 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
315}
316
317auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
318 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
319 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
320}
321
mflodmancc3d4422017-08-03 15:27:51322class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 18:45:46323 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23324 VideoStreamEncoderUnderTest(TimeController* time_controller,
325 TaskQueueFactory* task_queue_factory,
326 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 09:26:03327 const VideoStreamEncoderSettings& settings,
328 VideoStreamEncoder::BitrateAllocationCallbackType
329 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 12:31:23330 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 17:30:41331 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 13:03:05332 stats_proxy,
333 settings,
Yves Gerey665174f2018-06-19 13:03:05334 std::unique_ptr<OveruseFrameDetector>(
335 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 10:50:20336 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 09:26:03337 task_queue_factory,
338 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 12:31:23339 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 15:59:05340 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 11:02:36341 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 09:47:10342 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 14:29:22343 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 09:36:55344 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 14:29:22345 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 11:02:36346 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 15:26:01347 }
perkj803d97f2016-11-01 18:45:46348
Henrik Boström381d1092020-05-12 16:49:07349 void SetSourceAndWaitForRestrictionsUpdated(
350 rtc::VideoSourceInterface<VideoFrame>* source,
351 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 11:02:36352 FakeVideoSourceRestrictionsListener listener;
353 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07354 SetSource(source, degradation_preference);
355 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 11:02:36356 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 16:49:07357 }
358
359 void SetSourceAndWaitForFramerateUpdated(
360 rtc::VideoSourceInterface<VideoFrame>* source,
361 const DegradationPreference& degradation_preference) {
362 overuse_detector_proxy_->framerate_updated_event()->Reset();
363 SetSource(source, degradation_preference);
364 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
365 }
366
367 void OnBitrateUpdatedAndWaitForManagedResources(
368 DataRate target_bitrate,
369 DataRate stable_target_bitrate,
370 DataRate link_allocation,
371 uint8_t fraction_lost,
372 int64_t round_trip_time_ms,
373 double cwnd_reduce_ratio) {
374 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
375 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
376 // Bitrate is updated on the encoder queue.
377 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 16:49:07378 }
379
kthelgason2fc52542017-03-03 08:24:41380 // This is used as a synchronisation mechanism, to make sure that the
381 // encoder queue is not blocked before we start sending it frames.
382 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 07:43:50383 rtc::Event event;
Yves Gerey665174f2018-06-19 13:03:05384 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 08:24:41385 ASSERT_TRUE(event.Wait(5000));
386 }
387
Henrik Boström91aa7322020-04-28 10:24:33388 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 13:25:32389 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 10:24:33390 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12391 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36392 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33393 event.Set();
394 });
395 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 12:31:23396 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 10:24:33397 }
Tomas Gunnarsson612445e2020-09-21 12:31:23398
Henrik Boström91aa7322020-04-28 10:24:33399 void TriggerCpuUnderuse() {
400 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12401 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36402 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33403 event.Set();
404 });
405 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 12:31:23406 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 13:25:32407 }
kthelgason876222f2016-11-29 09:44:11408
Henrik Boström91aa7322020-04-28 10:24:33409 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 13:25:32410 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 10:24:33411 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36413 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 10:24:33414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 12:31:23417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 13:25:32418 }
Åsa Perssonb67c44c2019-09-24 13:25:32419 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 10:24:33420 rtc::Event event;
Evan Shrubsole85728412020-08-25 11:12:12421 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 11:02:36422 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 10:24:33423 event.Set();
424 });
425 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 12:31:23426 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 10:24:33427 }
428
Tomas Gunnarsson612445e2020-09-21 12:31:23429 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 09:27:48430 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 14:29:22431 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
432 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 11:02:36433 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 18:45:46434};
435
Noah Richards51db4212019-06-12 13:59:12436// Simulates simulcast behavior and makes highest stream resolutions divisible
437// by 4.
438class CroppingVideoStreamFactory
439 : public VideoEncoderConfig::VideoStreamFactoryInterface {
440 public:
Åsa Persson17b29b92020-10-17 10:57:58441 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 13:59:12442
443 private:
444 std::vector<VideoStream> CreateEncoderStreams(
445 int width,
446 int height,
447 const VideoEncoderConfig& encoder_config) override {
448 std::vector<VideoStream> streams = test::CreateVideoStreams(
449 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 13:59:12450 return streams;
451 }
Noah Richards51db4212019-06-12 13:59:12452};
453
sprangb1ca0732017-02-01 16:38:12454class AdaptingFrameForwarder : public test::FrameForwarder {
455 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23456 explicit AdaptingFrameForwarder(TimeController* time_controller)
457 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 12:51:49458 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 16:38:12459
460 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 11:13:32461 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12462 adaptation_enabled_ = enabled;
463 }
464
asaperssonfab67072017-04-04 12:51:49465 bool adaption_enabled() const {
Markus Handella3765182020-07-08 11:13:32466 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:12467 return adaptation_enabled_;
468 }
469
Henrik Boström1124ed12021-02-25 09:30:39470 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
471 // the resolution or frame rate was different than it is currently. If
472 // something else is modified, such as encoder resolutions, but the resolution
473 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-16 06:40:18474 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 11:13:32475 MutexLock lock(&mutex_);
asapersson09f05612017-05-16 06:40:18476 return last_wants_;
477 }
478
Danil Chapovalovb9b146c2018-06-15 10:28:07479 absl::optional<int> last_sent_width() const { return last_width_; }
480 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-09 01:04:29481
sprangb1ca0732017-02-01 16:38:12482 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 12:31:23483 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
484 time_controller_->AdvanceTime(TimeDelta::Millis(0));
485
sprangb1ca0732017-02-01 16:38:12486 int cropped_width = 0;
487 int cropped_height = 0;
488 int out_width = 0;
489 int out_height = 0;
sprangc5d62e22017-04-03 06:53:04490 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 12:31:23491 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
492 << "w=" << video_frame.width()
493 << "h=" << video_frame.height();
sprangc5d62e22017-04-03 06:53:04494 if (adapter_.AdaptFrameResolution(
495 video_frame.width(), video_frame.height(),
496 video_frame.timestamp_us() * 1000, &cropped_width,
497 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 22:49:37498 VideoFrame adapted_frame =
499 VideoFrame::Builder()
500 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
501 nullptr, out_width, out_height))
502 .set_timestamp_rtp(99)
503 .set_timestamp_ms(99)
504 .set_rotation(kVideoRotation_0)
505 .build();
sprangc5d62e22017-04-03 06:53:04506 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 15:54:17507 if (video_frame.has_update_rect()) {
508 adapted_frame.set_update_rect(
509 video_frame.update_rect().ScaleWithFrame(
510 video_frame.width(), video_frame.height(), 0, 0,
511 video_frame.width(), video_frame.height(), out_width,
512 out_height));
513 }
sprangc5d62e22017-04-03 06:53:04514 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-09 01:04:29515 last_width_.emplace(adapted_frame.width());
516 last_height_.emplace(adapted_frame.height());
517 } else {
Danil Chapovalovb9b146c2018-06-15 10:28:07518 last_width_ = absl::nullopt;
519 last_height_ = absl::nullopt;
sprangc5d62e22017-04-03 06:53:04520 }
sprangb1ca0732017-02-01 16:38:12521 } else {
Tomas Gunnarsson612445e2020-09-21 12:31:23522 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 16:38:12523 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-09 01:04:29524 last_width_.emplace(video_frame.width());
525 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 16:38:12526 }
527 }
528
529 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
530 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 11:13:32531 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 09:30:39532 rtc::VideoSinkWants prev_wants = sink_wants_locked();
533 bool did_adapt =
534 prev_wants.max_pixel_count != wants.max_pixel_count ||
535 prev_wants.target_pixel_count != wants.target_pixel_count ||
536 prev_wants.max_framerate_fps != wants.max_framerate_fps;
537 if (did_adapt) {
538 last_wants_ = prev_wants;
539 }
Rasmus Brandt287e4642019-11-15 15:56:01540 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 06:37:30541 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 16:38:12542 }
Tomas Gunnarsson612445e2020-09-21 12:31:23543
544 TimeController* const time_controller_;
sprangb1ca0732017-02-01 16:38:12545 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 11:13:32546 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
547 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 10:28:07548 absl::optional<int> last_width_;
549 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 16:38:12550};
sprangc5d62e22017-04-03 06:53:04551
Niels Möller213618e2018-07-24 07:29:58552// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-03 06:53:04553class MockableSendStatisticsProxy : public SendStatisticsProxy {
554 public:
555 MockableSendStatisticsProxy(Clock* clock,
556 const VideoSendStream::Config& config,
557 VideoEncoderConfig::ContentType content_type)
558 : SendStatisticsProxy(clock, config, content_type) {}
559
560 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 11:13:32561 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04562 if (mock_stats_)
563 return *mock_stats_;
564 return SendStatisticsProxy::GetStats();
565 }
566
Niels Möller213618e2018-07-24 07:29:58567 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 11:13:32568 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 07:29:58569 if (mock_stats_)
570 return mock_stats_->input_frame_rate;
571 return SendStatisticsProxy::GetInputFrameRate();
572 }
sprangc5d62e22017-04-03 06:53:04573 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 11:13:32574 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04575 mock_stats_.emplace(stats);
576 }
577
578 void ResetMockStats() {
Markus Handella3765182020-07-08 11:13:32579 MutexLock lock(&lock_);
sprangc5d62e22017-04-03 06:53:04580 mock_stats_.reset();
581 }
582
Tomas Gunnarsson612445e2020-09-21 12:31:23583 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
584 on_frame_dropped_ = std::move(callback);
585 }
586
sprangc5d62e22017-04-03 06:53:04587 private:
Tomas Gunnarsson612445e2020-09-21 12:31:23588 void OnFrameDropped(DropReason reason) override {
589 SendStatisticsProxy::OnFrameDropped(reason);
590 if (on_frame_dropped_)
591 on_frame_dropped_(reason);
592 }
593
Markus Handella3765182020-07-08 11:13:32594 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 10:28:07595 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 12:31:23596 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-03 06:53:04597};
598
philipel9b058032020-02-10 10:30:00599class MockEncoderSelector
600 : public VideoEncoderFactory::EncoderSelectorInterface {
601 public:
Danil Chapovalov91fdc602020-05-14 17:17:51602 MOCK_METHOD(void,
603 OnCurrentEncoder,
604 (const SdpVideoFormat& format),
605 (override));
606 MOCK_METHOD(absl::optional<SdpVideoFormat>,
607 OnAvailableBitrate,
608 (const DataRate& rate),
609 (override));
610 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 10:30:00611};
612
perkj803d97f2016-11-01 18:45:46613} // namespace
614
mflodmancc3d4422017-08-03 15:27:51615class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 08:17:40616 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23617 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 08:17:40618
mflodmancc3d4422017-08-03 15:27:51619 VideoStreamEncoderTest()
perkj26091b12016-09-01 08:17:40620 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-03 06:45:26621 codec_width_(320),
622 codec_height_(240),
Åsa Persson8c1bf952018-09-13 08:42:19623 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 12:31:23624 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 07:04:13625 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-03 06:53:04626 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 12:31:23627 time_controller_.GetClock(),
perkj803d97f2016-11-01 18:45:46628 video_send_config_,
629 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 12:31:23630 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 08:17:40631
632 void SetUp() override {
perkj803d97f2016-11-01 18:45:46633 metrics::Reset();
perkj26091b12016-09-01 08:17:40634 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 07:04:13635 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56636 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 12:18:34637 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 13:36:51638 video_send_config_.rtp.payload_name = "FAKE";
639 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 08:17:40640
Per512ecb32016-09-23 13:52:06641 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51642 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 06:57:51643 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
644 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
645 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 15:41:30646 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 14:06:52647
Niels Möllerf1338562018-04-26 07:51:47648 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 12:37:00649 }
650
Per Kjellanderb03b6c82021-01-03 09:26:03651 void ConfigureEncoder(
652 VideoEncoderConfig video_encoder_config,
653 VideoStreamEncoder::BitrateAllocationCallbackType
654 allocation_callback_type =
655 VideoStreamEncoder::BitrateAllocationCallbackType::
656 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 15:27:51657 if (video_stream_encoder_)
658 video_stream_encoder_->Stop();
659 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 12:31:23660 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 09:26:03661 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 15:27:51662 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
663 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:41664 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 15:27:51665 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
666 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:47667 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:51668 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 12:37:00669 }
670
Per Kjellanderb03b6c82021-01-03 09:26:03671 void ResetEncoder(const std::string& payload_name,
672 size_t num_streams,
673 size_t num_temporal_layers,
674 unsigned char num_spatial_layers,
675 bool screenshare,
676 VideoStreamEncoder::BitrateAllocationCallbackType
677 allocation_callback_type =
678 VideoStreamEncoder::BitrateAllocationCallbackType::
679 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 13:36:51680 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 12:37:00681
682 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:51683 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
684 num_streams, &video_encoder_config);
685 for (auto& layer : video_encoder_config.simulcast_layers) {
686 layer.num_temporal_layers = num_temporal_layers;
687 layer.max_framerate = kDefaultFramerate;
688 }
Erik Språngd7329ca2019-02-21 20:19:53689 video_encoder_config.max_bitrate_bps =
690 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 14:06:52691 video_encoder_config.content_type =
692 screenshare ? VideoEncoderConfig::ContentType::kScreen
693 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 07:28:40694 if (payload_name == "VP9") {
695 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
696 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 13:29:23697 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 07:28:40698 video_encoder_config.encoder_specific_settings =
699 new rtc::RefCountedObject<
700 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
701 }
Per Kjellanderb03b6c82021-01-03 09:26:03702 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 08:17:40703 }
704
sprang57c2fff2017-01-16 14:24:02705 VideoFrame CreateFrame(int64_t ntp_time_ms,
706 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 22:49:37707 VideoFrame frame =
708 VideoFrame::Builder()
709 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
710 destruction_event, codec_width_, codec_height_))
711 .set_timestamp_rtp(99)
712 .set_timestamp_ms(99)
713 .set_rotation(kVideoRotation_0)
714 .build();
sprang57c2fff2017-01-16 14:24:02715 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 08:17:40716 return frame;
717 }
718
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26719 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
720 rtc::Event* destruction_event,
721 int offset_x) const {
722 VideoFrame frame =
723 VideoFrame::Builder()
724 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
725 destruction_event, codec_width_, codec_height_))
726 .set_timestamp_rtp(99)
727 .set_timestamp_ms(99)
728 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 09:34:12729 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26730 .build();
731 frame.set_ntp_time_ms(ntp_time_ms);
732 return frame;
733 }
734
sprang57c2fff2017-01-16 14:24:02735 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 22:49:37736 VideoFrame frame =
737 VideoFrame::Builder()
738 .set_video_frame_buffer(
739 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
740 .set_timestamp_rtp(99)
741 .set_timestamp_ms(99)
742 .set_rotation(kVideoRotation_0)
743 .build();
sprang57c2fff2017-01-16 14:24:02744 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-03 06:53:04745 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 18:45:46746 return frame;
747 }
748
Evan Shrubsole895556e2020-10-05 07:15:13749 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
750 VideoFrame frame =
751 VideoFrame::Builder()
752 .set_video_frame_buffer(NV12Buffer::Create(width, height))
753 .set_timestamp_rtp(99)
754 .set_timestamp_ms(99)
755 .set_rotation(kVideoRotation_0)
756 .build();
757 frame.set_ntp_time_ms(ntp_time_ms);
758 frame.set_timestamp_us(ntp_time_ms * 1000);
759 return frame;
760 }
761
Noah Richards51db4212019-06-12 13:59:12762 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
763 rtc::Event* destruction_event,
764 int width,
765 int height) const {
766 VideoFrame frame =
767 VideoFrame::Builder()
768 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
769 destruction_event, width, height))
770 .set_timestamp_rtp(99)
771 .set_timestamp_ms(99)
772 .set_rotation(kVideoRotation_0)
773 .build();
774 frame.set_ntp_time_ms(ntp_time_ms);
775 return frame;
776 }
777
Evan Shrubsole895556e2020-10-05 07:15:13778 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
779 rtc::Event* destruction_event,
780 int width,
781 int height) const {
782 VideoFrame frame = VideoFrame::Builder()
783 .set_video_frame_buffer(
784 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
785 destruction_event, width, height))
786 .set_timestamp_rtp(99)
787 .set_timestamp_ms(99)
788 .set_rotation(kVideoRotation_0)
789 .build();
790 frame.set_ntp_time_ms(ntp_time_ms);
791 return frame;
792 }
793
Noah Richards51db4212019-06-12 13:59:12794 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
795 rtc::Event* destruction_event) const {
796 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
797 codec_height_);
798 }
799
Åsa Perssonc29cb2c2019-03-25 11:06:59800 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 16:49:07801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:07802 DataRate::BitsPerSec(kTargetBitrateBps),
803 DataRate::BitsPerSec(kTargetBitrateBps),
804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 11:06:59805
806 video_source_.IncomingCapturedFrame(
807 CreateFrame(1, codec_width_, codec_height_));
808 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 13:09:05809 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 11:06:59810 }
811
sprang4847ae62017-06-27 14:06:52812 void WaitForEncodedFrame(int64_t expected_ntp_time) {
813 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 12:31:23814 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52815 }
816
817 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
818 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:23819 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52820 return ok;
821 }
822
823 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
824 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 12:31:23825 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52826 }
827
828 void ExpectDroppedFrame() {
829 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 12:31:23830 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52831 }
832
833 bool WaitForFrame(int64_t timeout_ms) {
834 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 12:31:23835 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52836 return ok;
837 }
838
perkj26091b12016-09-01 08:17:40839 class TestEncoder : public test::FakeEncoder {
840 public:
Tomas Gunnarsson612445e2020-09-21 12:31:23841 explicit TestEncoder(TimeController* time_controller)
842 : FakeEncoder(time_controller->GetClock()),
843 time_controller_(time_controller) {
844 RTC_DCHECK(time_controller_);
845 }
perkj26091b12016-09-01 08:17:40846
asaperssonfab67072017-04-04 12:51:49847 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 11:13:32848 MutexLock lock(&mutex_);
perkjfa10b552016-10-03 06:45:26849 return config_;
850 }
851
852 void BlockNextEncode() {
Markus Handella3765182020-07-08 11:13:32853 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-03 06:45:26854 block_next_encode_ = true;
855 }
856
Erik Språngaed30702018-11-05 11:57:17857 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 11:13:32858 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 15:44:42859 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 14:52:33860 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 11:42:18861 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 14:56:00862 info.scaling_settings = VideoEncoder::ScalingSettings(
863 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 11:42:18864 }
865 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 11:06:59866 for (int i = 0; i < kMaxSpatialLayers; ++i) {
867 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 14:27:35868 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 11:06:59869 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 14:27:35870 for (int tid = 0; tid < num_layers; ++tid)
871 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 11:06:59872 }
873 }
Erik Språngaed30702018-11-05 11:57:17874 }
Sergey Silkin6456e352019-07-08 15:56:40875
876 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 08:47:11877 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 15:50:00878 info.apply_alignment_to_all_simulcast_layers =
879 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 12:56:45880 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 11:57:17881 return info;
kthelgason876222f2016-11-29 09:44:11882 }
883
Erik Språngb7cb7b52019-02-26 14:52:33884 int32_t RegisterEncodeCompleteCallback(
885 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 11:13:32886 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:33887 encoded_image_callback_ = callback;
888 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
889 }
890
perkjfa10b552016-10-03 06:45:26891 void ContinueEncode() { continue_encode_event_.Set(); }
892
893 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
894 uint32_t timestamp) const {
Markus Handella3765182020-07-08 11:13:32895 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-03 06:45:26896 EXPECT_EQ(timestamp_, timestamp);
897 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
898 }
899
kthelgason2fc52542017-03-03 08:24:41900 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 11:13:32901 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 08:24:41902 quality_scaling_ = b;
903 }
kthelgasonad9010c2017-02-14 08:46:51904
Rasmus Brandt5cad55b2019-12-19 08:47:11905 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 11:13:32906 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 08:47:11907 requested_resolution_alignment_ = requested_resolution_alignment;
908 }
909
Åsa Perssonc5a74ff2020-09-20 15:50:00910 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
911 MutexLock lock(&local_mutex_);
912 apply_alignment_to_all_simulcast_layers_ = b;
913 }
914
Mirta Dvornicicccc1b572019-01-15 11:42:18915 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 11:13:32916 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 11:42:18917 is_hardware_accelerated_ = is_hardware_accelerated;
918 }
919
Åsa Perssonc29cb2c2019-03-25 11:06:59920 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
921 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 11:13:32922 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:59923 temporal_layers_supported_[spatial_idx] = supported;
924 }
925
Sergey Silkin6456e352019-07-08 15:56:40926 void SetResolutionBitrateLimits(
927 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 11:13:32928 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 15:56:40929 resolution_bitrate_limits_ = thresholds;
930 }
931
sprangfe627f32017-03-29 15:24:59932 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 11:13:32933 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 15:24:59934 force_init_encode_failed_ = force_failure;
935 }
936
Niels Möller6bb5ab92019-01-11 10:11:10937 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 11:13:32938 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:10939 rate_factor_ = rate_factor;
940 }
941
Erik Språngd7329ca2019-02-21 20:19:53942 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 11:13:32943 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 10:11:10944 return last_framerate_;
945 }
946
Erik Språngd7329ca2019-02-21 20:19:53947 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 11:13:32948 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26949 return last_update_rect_;
950 }
951
Niels Möller87e2d782019-03-07 09:18:23952 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 11:13:32953 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:53954 return last_frame_types_;
955 }
956
957 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 09:18:23958 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 14:43:58959 keyframe ? VideoFrameType::kVideoFrameKey
960 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:53961 {
Markus Handella3765182020-07-08 11:13:32962 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:53963 last_frame_types_ = frame_type;
964 }
Niels Möllerb859b322019-03-07 11:40:01965 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 20:19:53966 }
967
Sergey Silkin0e42cf72021-03-15 09:12:57968 void InjectEncodedImage(const EncodedImage& image,
969 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 11:13:32970 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 09:12:57971 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 14:52:33972 }
973
Mirta Dvornicic97910da2020-07-14 13:29:23974 void SetEncodedImageData(
975 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 11:13:32976 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 13:29:23977 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 14:30:16978 }
979
Erik Språngd7329ca2019-02-21 20:19:53980 void ExpectNullFrame() {
Markus Handella3765182020-07-08 11:13:32981 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:53982 expect_null_frame_ = true;
983 }
984
Erik Språng5056af02019-09-02 13:53:11985 absl::optional<VideoEncoder::RateControlParameters>
986 GetAndResetLastRateControlSettings() {
987 auto settings = last_rate_control_settings_;
988 last_rate_control_settings_.reset();
989 return settings;
Erik Språngd7329ca2019-02-21 20:19:53990 }
991
Evan Shrubsole895556e2020-10-05 07:15:13992 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
993 MutexLock lock(&local_mutex_);
994 return last_input_pixel_format_;
995 }
996
Sergey Silkin5ee69672019-07-02 12:18:34997 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 11:13:32998 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 12:18:34999 return num_encoder_initializations_;
1000 }
1001
Evan Shrubsole7c079f62019-09-26 07:55:031002 int GetNumSetRates() const {
Markus Handella3765182020-07-08 11:13:321003 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031004 return num_set_rates_;
1005 }
1006
Åsa Perssonc5a74ff2020-09-20 15:50:001007 VideoCodec video_codec() const {
1008 MutexLock lock(&local_mutex_);
1009 return video_codec_;
1010 }
1011
Evan Shrubsoleb556b082020-10-08 12:56:451012 void SetPreferredPixelFormats(
1013 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1014 pixel_formats) {
1015 MutexLock lock(&local_mutex_);
1016 preferred_pixel_formats_ = std::move(pixel_formats);
1017 }
1018
perkjfa10b552016-10-03 06:45:261019 private:
perkj26091b12016-09-01 08:17:401020 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 09:18:231021 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 08:17:401022 bool block_encode;
1023 {
Markus Handella3765182020-07-08 11:13:321024 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 20:19:531025 if (expect_null_frame_) {
1026 EXPECT_EQ(input_image.timestamp(), 0u);
1027 EXPECT_EQ(input_image.width(), 1);
1028 last_frame_types_ = *frame_types;
1029 expect_null_frame_ = false;
1030 } else {
1031 EXPECT_GT(input_image.timestamp(), timestamp_);
1032 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1033 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1034 }
perkj26091b12016-09-01 08:17:401035
1036 timestamp_ = input_image.timestamp();
1037 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 18:45:461038 last_input_width_ = input_image.width();
1039 last_input_height_ = input_image.height();
perkj26091b12016-09-01 08:17:401040 block_encode = block_next_encode_;
1041 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:261042 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 20:19:531043 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 07:15:131044 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 08:17:401045 }
Niels Möllerb859b322019-03-07 11:40:011046 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 08:17:401047 if (block_encode)
perkja49cbd32016-09-16 14:53:411048 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 12:31:231049
perkj26091b12016-09-01 08:17:401050 return result;
1051 }
1052
Niels Möller08ae7ce2020-09-23 13:58:121053 CodecSpecificInfo EncodeHook(
1054 EncodedImage& encoded_image,
1055 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 15:30:361056 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231057 {
1058 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 15:30:361059 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 13:29:231060 }
1061 MutexLock lock(&local_mutex_);
1062 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 15:30:361063 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 13:29:231064 }
Danil Chapovalov2549f172020-08-12 15:30:361065 return codec_specific;
Mirta Dvornicic97910da2020-07-14 13:29:231066 }
1067
sprangfe627f32017-03-29 15:24:591068 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 12:57:571069 const Settings& settings) override {
1070 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 12:18:341071
Markus Handella3765182020-07-08 11:13:321072 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331073 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 12:18:341074
1075 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 15:50:001076 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 12:18:341077
Erik Språng82fad3d2018-03-21 08:57:231078 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 15:24:591079 // Simulate setting up temporal layers, in order to validate the life
1080 // cycle of these objects.
Elad Aloncde8ab22019-03-20 10:56:201081 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 09:20:091082 frame_buffer_controller_ =
1083 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 15:24:591084 }
Erik Språngb7cb7b52019-02-26 14:52:331085 if (force_init_encode_failed_) {
1086 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 15:24:591087 return -1;
Erik Språngb7cb7b52019-02-26 14:52:331088 }
Mirta Dvornicicccc1b572019-01-15 11:42:181089
Erik Språngb7cb7b52019-02-26 14:52:331090 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 15:24:591091 return res;
1092 }
1093
Erik Språngb7cb7b52019-02-26 14:52:331094 int32_t Release() override {
Markus Handella3765182020-07-08 11:13:321095 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331096 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1097 initialized_ = EncoderState::kUninitialized;
1098 return FakeEncoder::Release();
1099 }
1100
Erik Språng16cb8f52019-04-12 11:59:091101 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 11:13:321102 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 07:55:031103 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 10:11:101104 VideoBitrateAllocation adjusted_rate_allocation;
1105 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1106 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 11:59:091107 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 10:11:101108 adjusted_rate_allocation.SetBitrate(
1109 si, ti,
Erik Språng16cb8f52019-04-12 11:59:091110 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 10:11:101111 rate_factor_));
1112 }
1113 }
1114 }
Erik Språng16cb8f52019-04-12 11:59:091115 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 13:53:111116 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 11:59:091117 RateControlParameters adjusted_paramters = parameters;
1118 adjusted_paramters.bitrate = adjusted_rate_allocation;
1119 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 10:11:101120 }
1121
Tomas Gunnarsson612445e2020-09-21 12:31:231122 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321123 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 14:52:331124 enum class EncoderState {
1125 kUninitialized,
1126 kInitializationFailed,
1127 kInitialized
Markus Handella3765182020-07-08 11:13:321128 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1129 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 08:17:401130 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 11:13:321131 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1132 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1133 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1134 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1135 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1136 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 15:50:001137 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1138 false;
Markus Handella3765182020-07-08 11:13:321139 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 13:29:231140 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1141 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 10:56:201142 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 11:13:321143 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 11:06:591144 absl::optional<bool>
1145 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 11:13:321146 local_mutex_);
1147 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1148 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1149 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 13:53:111150 absl::optional<VideoEncoder::RateControlParameters>
1151 last_rate_control_settings_;
Markus Handella3765182020-07-08 11:13:321152 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1153 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 09:18:231154 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 20:19:531155 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 11:13:321156 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1157 nullptr;
Evan Shrubsolefb862742020-03-16 15:18:361158 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 11:13:321159 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 15:56:401160 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 11:13:321161 RTC_GUARDED_BY(local_mutex_);
1162 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 15:50:001163 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 07:15:131164 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1165 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 12:56:451166 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1167 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 08:17:401168 };
1169
mflodmancc3d4422017-08-03 15:27:511170 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 08:17:401171 public:
Tomas Gunnarsson612445e2020-09-21 12:31:231172 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1173 : time_controller_(time_controller), test_encoder_(test_encoder) {
1174 RTC_DCHECK(time_controller_);
1175 }
perkj26091b12016-09-01 08:17:401176
perkj26091b12016-09-01 08:17:401177 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 14:06:521178 EXPECT_TRUE(
1179 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1180 }
1181
1182 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1183 int64_t timeout_ms) {
perkj26091b12016-09-01 08:17:401184 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:231185 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 14:06:521186 return false;
perkj26091b12016-09-01 08:17:401187 {
Markus Handella3765182020-07-08 11:13:321188 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121189 timestamp = last_timestamp_;
perkj26091b12016-09-01 08:17:401190 }
1191 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 14:06:521192 return true;
perkj26091b12016-09-01 08:17:401193 }
1194
sprangb1ca0732017-02-01 16:38:121195 void WaitForEncodedFrame(uint32_t expected_width,
1196 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 12:31:231197 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 13:13:561198 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-03 06:53:041199 }
1200
Åsa Perssonc74d8da2017-12-04 13:13:561201 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-03 06:53:041202 uint32_t expected_height) {
sprangb1ca0732017-02-01 16:38:121203 uint32_t width = 0;
1204 uint32_t height = 0;
sprangb1ca0732017-02-01 16:38:121205 {
Markus Handella3765182020-07-08 11:13:321206 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 16:38:121207 width = last_width_;
1208 height = last_height_;
1209 }
1210 EXPECT_EQ(expected_height, height);
1211 EXPECT_EQ(expected_width, width);
1212 }
1213
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141214 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1215 VideoRotation rotation;
1216 {
Markus Handella3765182020-07-08 11:13:321217 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141218 rotation = last_rotation_;
1219 }
1220 EXPECT_EQ(expected_rotation, rotation);
1221 }
1222
Tomas Gunnarsson612445e2020-09-21 12:31:231223 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 15:02:221224
sprangc5d62e22017-04-03 06:53:041225 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 12:31:231226 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1227 bool ret = encoded_frame_event_.Wait(timeout_ms);
1228 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1229 return ret;
sprangc5d62e22017-04-03 06:53:041230 }
1231
perkj26091b12016-09-01 08:17:401232 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 11:13:321233 MutexLock lock(&mutex_);
perkj26091b12016-09-01 08:17:401234 expect_frames_ = false;
1235 }
1236
asaperssonfab67072017-04-04 12:51:491237 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 11:13:321238 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061239 return number_of_reconfigurations_;
1240 }
1241
asaperssonfab67072017-04-04 12:51:491242 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 11:13:321243 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061244 return min_transmit_bitrate_bps_;
1245 }
1246
Erik Språngd7329ca2019-02-21 20:19:531247 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 11:13:321248 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 20:19:531249 num_expected_layers_ = num_layers;
1250 }
1251
Erik Språngb7cb7b52019-02-26 14:52:331252 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 11:13:321253 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 14:52:331254 return last_capture_time_ms_;
1255 }
1256
Sergey Silkin0e42cf72021-03-15 09:12:571257 const EncodedImage& GetLastEncodedImage() {
1258 MutexLock lock(&mutex_);
1259 return last_encoded_image_;
1260 }
1261
Mirta Dvornicic28f0eb22019-05-28 14:30:161262 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 11:13:321263 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 14:30:161264 return std::move(last_encoded_image_data_);
1265 }
1266
Per Kjellanderdcef6412020-10-07 13:09:051267 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1268 MutexLock lock(&mutex_);
1269 return last_bitrate_allocation_;
1270 }
1271
1272 int number_of_bitrate_allocations() const {
1273 MutexLock lock(&mutex_);
1274 return number_of_bitrate_allocations_;
1275 }
1276
Per Kjellandera9434842020-10-15 15:53:221277 VideoLayersAllocation GetLastVideoLayersAllocation() {
1278 MutexLock lock(&mutex_);
1279 return last_layers_allocation_;
1280 }
1281
1282 int number_of_layers_allocations() const {
1283 MutexLock lock(&mutex_);
1284 return number_of_layers_allocations_;
1285 }
1286
perkj26091b12016-09-01 08:17:401287 private:
sergeyu2cb155a2016-11-04 18:39:291288 Result OnEncodedImage(
1289 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 15:30:361290 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 11:13:321291 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061292 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 09:12:571293 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 14:30:161294 last_encoded_image_data_ = std::vector<uint8_t>(
1295 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 20:19:531296 uint32_t timestamp = encoded_image.Timestamp();
1297 if (last_timestamp_ != timestamp) {
1298 num_received_layers_ = 1;
1299 } else {
1300 ++num_received_layers_;
1301 }
1302 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 14:52:331303 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 16:38:121304 last_width_ = encoded_image._encodedWidth;
1305 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141306 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 20:19:531307 if (num_received_layers_ == num_expected_layers_) {
1308 encoded_frame_event_.Set();
1309 }
sprangb1ca0732017-02-01 16:38:121310 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 13:52:061311 }
1312
Rasmus Brandtc402dbe2019-02-04 10:09:461313 void OnEncoderConfigurationChanged(
1314 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 12:10:271315 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 10:09:461316 VideoEncoderConfig::ContentType content_type,
1317 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 11:13:321318 MutexLock lock(&mutex_);
Per512ecb32016-09-23 13:52:061319 ++number_of_reconfigurations_;
1320 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1321 }
1322
Per Kjellanderdcef6412020-10-07 13:09:051323 void OnBitrateAllocationUpdated(
1324 const VideoBitrateAllocation& allocation) override {
1325 MutexLock lock(&mutex_);
1326 ++number_of_bitrate_allocations_;
1327 last_bitrate_allocation_ = allocation;
1328 }
1329
Per Kjellandera9434842020-10-15 15:53:221330 void OnVideoLayersAllocationUpdated(
1331 VideoLayersAllocation allocation) override {
1332 MutexLock lock(&mutex_);
1333 ++number_of_layers_allocations_;
1334 last_layers_allocation_ = allocation;
1335 rtc::StringBuilder log;
1336 for (const auto& layer : allocation.active_spatial_layers) {
1337 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1338 << "[";
1339 for (const auto target_bitrate :
1340 layer.target_bitrate_per_temporal_layer) {
1341 log << target_bitrate.kbps() << ",";
1342 }
1343 log << "]";
1344 }
1345 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1346 }
1347
Tomas Gunnarsson612445e2020-09-21 12:31:231348 TimeController* const time_controller_;
Markus Handella3765182020-07-08 11:13:321349 mutable Mutex mutex_;
perkj26091b12016-09-01 08:17:401350 TestEncoder* test_encoder_;
1351 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 09:12:571352 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 14:30:161353 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 16:38:121354 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 14:52:331355 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 16:38:121356 uint32_t last_height_ = 0;
1357 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141358 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 20:19:531359 size_t num_expected_layers_ = 1;
1360 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 08:17:401361 bool expect_frames_ = true;
Per512ecb32016-09-23 13:52:061362 int number_of_reconfigurations_ = 0;
1363 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 13:09:051364 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1365 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 15:53:221366 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1367 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 08:17:401368 };
1369
Sergey Silkin5ee69672019-07-02 12:18:341370 class VideoBitrateAllocatorProxyFactory
1371 : public VideoBitrateAllocatorFactory {
1372 public:
1373 VideoBitrateAllocatorProxyFactory()
1374 : bitrate_allocator_factory_(
1375 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1376
1377 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1378 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 11:13:321379 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341380 codec_config_ = codec;
1381 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1382 }
1383
1384 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 11:13:321385 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341386 return codec_config_;
1387 }
1388
1389 private:
1390 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1391
Markus Handella3765182020-07-08 11:13:321392 mutable Mutex mutex_;
1393 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 12:18:341394 };
1395
Tomas Gunnarsson612445e2020-09-21 12:31:231396 Clock* clock() { return time_controller_.GetClock(); }
1397 void AdvanceTime(TimeDelta duration) {
1398 time_controller_.AdvanceTime(duration);
1399 }
1400
1401 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1402
1403 protected:
1404 virtual TaskQueueFactory* GetTaskQueueFactory() {
1405 return time_controller_.GetTaskQueueFactory();
1406 }
1407
1408 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 08:17:401409 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 15:41:301410 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 13:52:061411 int codec_width_;
1412 int codec_height_;
sprang4847ae62017-06-27 14:06:521413 int max_framerate_;
perkj26091b12016-09-01 08:17:401414 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 07:07:241415 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 12:18:341416 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-03 06:53:041417 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 08:17:401418 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 12:31:231419 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 15:27:511420 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 08:17:401421};
1422
mflodmancc3d4422017-08-03 15:27:511423TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 16:49:071424 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071425 DataRate::BitsPerSec(kTargetBitrateBps),
1426 DataRate::BitsPerSec(kTargetBitrateBps),
1427 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 07:43:501428 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411429 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 14:06:521430 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 14:53:411431 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 15:27:511432 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401433}
1434
mflodmancc3d4422017-08-03 15:27:511435TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 08:17:401436 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 07:43:501437 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 11:05:491438 // The encoder will cache up to one frame for a short duration. Adding two
1439 // frames means that the first frame will be dropped and the second frame will
1440 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 14:53:411441 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 12:31:231442 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 11:05:491443 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 14:53:411444 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401445
Henrik Boström381d1092020-05-12 16:49:071446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071447 DataRate::BitsPerSec(kTargetBitrateBps),
1448 DataRate::BitsPerSec(kTargetBitrateBps),
1449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401450
Sebastian Janssona3177052018-04-10 11:05:491451 // The pending frame should be received.
sprang4847ae62017-06-27 14:06:521452 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 11:05:491453 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1454
1455 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:511456 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401457}
1458
mflodmancc3d4422017-08-03 15:27:511459TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 16:49:071460 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071461 DataRate::BitsPerSec(kTargetBitrateBps),
1462 DataRate::BitsPerSec(kTargetBitrateBps),
1463 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 14:53:411464 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521465 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401466
Henrik Boström381d1092020-05-12 16:49:071467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1468 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1469 0, 0, 0);
Sebastian Janssona3177052018-04-10 11:05:491470 // The encoder will cache up to one frame for a short duration. Adding two
1471 // frames means that the first frame will be dropped and the second frame will
1472 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 14:53:411473 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 11:05:491474 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401475
Henrik Boström381d1092020-05-12 16:49:071476 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071477 DataRate::BitsPerSec(kTargetBitrateBps),
1478 DataRate::BitsPerSec(kTargetBitrateBps),
1479 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:521480 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 11:05:491481 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1482 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 15:27:511483 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401484}
1485
mflodmancc3d4422017-08-03 15:27:511486TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 16:49:071487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071488 DataRate::BitsPerSec(kTargetBitrateBps),
1489 DataRate::BitsPerSec(kTargetBitrateBps),
1490 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 14:53:411491 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521492 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401493
1494 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 14:53:411495 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 08:17:401496
perkja49cbd32016-09-16 14:53:411497 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521498 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511499 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401500}
1501
mflodmancc3d4422017-08-03 15:27:511502TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 16:49:071503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071504 DataRate::BitsPerSec(kTargetBitrateBps),
1505 DataRate::BitsPerSec(kTargetBitrateBps),
1506 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401507
perkja49cbd32016-09-16 14:53:411508 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521509 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401510
mflodmancc3d4422017-08-03 15:27:511511 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401512 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 07:43:501513 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411514 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1515 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401516}
1517
Tomas Gunnarsson612445e2020-09-21 12:31:231518class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1519 public:
1520 VideoStreamEncoderBlockedTest() {}
1521
1522 TaskQueueFactory* GetTaskQueueFactory() override {
1523 return task_queue_factory_.get();
1524 }
1525
1526 private:
1527 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1528 CreateDefaultTaskQueueFactory();
1529};
1530
1531TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 16:49:071532 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071533 DataRate::BitsPerSec(kTargetBitrateBps),
1534 DataRate::BitsPerSec(kTargetBitrateBps),
1535 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401536
Tomas Gunnarsson612445e2020-09-21 12:31:231537 int dropped_count = 0;
1538 stats_proxy_->SetDroppedFrameCallback(
1539 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1540 ++dropped_count;
1541 });
1542
perkj26091b12016-09-01 08:17:401543 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 14:53:411544 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521545 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401546 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1547 // call to ContinueEncode.
perkja49cbd32016-09-16 14:53:411548 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1549 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401550 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 14:06:521551 WaitForEncodedFrame(3);
perkj26091b12016-09-01 08:17:401552
mflodmancc3d4422017-08-03 15:27:511553 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 12:31:231554
1555 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 08:17:401556}
1557
Noah Richards51db4212019-06-12 13:59:121558TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 16:49:071559 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071560 DataRate::BitsPerSec(kTargetBitrateBps),
1561 DataRate::BitsPerSec(kTargetBitrateBps),
1562 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121563
1564 rtc::Event frame_destroyed_event;
1565 video_source_.IncomingCapturedFrame(
1566 CreateFakeNativeFrame(1, &frame_destroyed_event));
1567 ExpectDroppedFrame();
1568 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1569 video_stream_encoder_->Stop();
1570}
1571
1572TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1573 // Use the cropping factory.
1574 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 10:57:581575 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 13:59:121576 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1577 kMaxPayloadLength);
1578 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1579
1580 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 16:49:071581 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071582 DataRate::BitsPerSec(kTargetBitrateBps),
1583 DataRate::BitsPerSec(kTargetBitrateBps),
1584 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121585 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1586 WaitForEncodedFrame(1);
1587 // The encoder will have been configured once.
1588 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1589 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1590 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1591
1592 // Now send in a fake frame that needs to be cropped as the width/height
1593 // aren't divisible by 4 (see CreateEncoderStreams above).
1594 rtc::Event frame_destroyed_event;
1595 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1596 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1597 ExpectDroppedFrame();
1598 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1599 video_stream_encoder_->Stop();
1600}
1601
Evan Shrubsole895556e2020-10-05 07:15:131602TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1604 DataRate::BitsPerSec(kTargetBitrateBps),
1605 DataRate::BitsPerSec(kTargetBitrateBps),
1606 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1607
1608 video_source_.IncomingCapturedFrame(
1609 CreateNV12Frame(1, codec_width_, codec_height_));
1610 WaitForEncodedFrame(1);
1611 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1612 fake_encoder_.GetLastInputPixelFormat());
1613 video_stream_encoder_->Stop();
1614}
1615
Evan Shrubsoleb556b082020-10-08 12:56:451616TEST_F(VideoStreamEncoderTest,
1617 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1618 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1619 DataRate::BitsPerSec(kTargetBitrateBps),
1620 DataRate::BitsPerSec(kTargetBitrateBps),
1621 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1622
1623 fake_encoder_.SetPreferredPixelFormats({});
1624
1625 rtc::Event frame_destroyed_event;
1626 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1627 1, &frame_destroyed_event, codec_width_, codec_height_));
1628 WaitForEncodedFrame(1);
1629 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1630 fake_encoder_.GetLastInputPixelFormat());
1631 video_stream_encoder_->Stop();
1632}
1633
1634TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1636 DataRate::BitsPerSec(kTargetBitrateBps),
1637 DataRate::BitsPerSec(kTargetBitrateBps),
1638 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1639
1640 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1641
1642 rtc::Event frame_destroyed_event;
1643 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1644 1, &frame_destroyed_event, codec_width_, codec_height_));
1645 WaitForEncodedFrame(1);
1646 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1647 fake_encoder_.GetLastInputPixelFormat());
1648 video_stream_encoder_->Stop();
1649}
1650
1651TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1653 DataRate::BitsPerSec(kTargetBitrateBps),
1654 DataRate::BitsPerSec(kTargetBitrateBps),
1655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1656
1657 // Fake NV12 native frame does not allow mapping to I444.
1658 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1659
1660 rtc::Event frame_destroyed_event;
1661 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1662 1, &frame_destroyed_event, codec_width_, codec_height_));
1663 WaitForEncodedFrame(1);
1664 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1665 fake_encoder_.GetLastInputPixelFormat());
1666 video_stream_encoder_->Stop();
1667}
1668
Evan Shrubsole895556e2020-10-05 07:15:131669TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1671 DataRate::BitsPerSec(kTargetBitrateBps),
1672 DataRate::BitsPerSec(kTargetBitrateBps),
1673 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1674
1675 rtc::Event frame_destroyed_event;
1676 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1677 1, &frame_destroyed_event, codec_width_, codec_height_));
1678 WaitForEncodedFrame(1);
1679 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1680 fake_encoder_.GetLastInputPixelFormat());
1681 video_stream_encoder_->Stop();
1682}
1683
Ying Wang9b881ab2020-02-07 13:29:321684TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 16:49:071685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071686 DataRate::BitsPerSec(kTargetBitrateBps),
1687 DataRate::BitsPerSec(kTargetBitrateBps),
1688 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 13:29:321689 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1690 WaitForEncodedFrame(1);
1691
Henrik Boström381d1092020-05-12 16:49:071692 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071693 DataRate::BitsPerSec(kTargetBitrateBps),
1694 DataRate::BitsPerSec(kTargetBitrateBps),
1695 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 13:29:321696 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1697 // frames. Adding two frames means that the first frame will be dropped and
1698 // the second frame will be sent to the encoder.
1699 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1700 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1701 WaitForEncodedFrame(3);
1702 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1703 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1704 WaitForEncodedFrame(5);
1705 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1706 video_stream_encoder_->Stop();
1707}
1708
mflodmancc3d4422017-08-03 15:27:511709TEST_F(VideoStreamEncoderTest,
1710 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 16:49:071711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071712 DataRate::BitsPerSec(kTargetBitrateBps),
1713 DataRate::BitsPerSec(kTargetBitrateBps),
1714 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 20:37:571715 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061716
1717 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551718 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521719 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571720 // The encoder will have been configured once when the first frame is
1721 // received.
1722 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061723
1724 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:511725 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 13:52:061726 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 15:27:511727 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:471728 kMaxPayloadLength);
Per512ecb32016-09-23 13:52:061729
1730 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551731 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521732 WaitForEncodedFrame(2);
Per21d45d22016-10-30 20:37:571733 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-30 06:25:401734 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-30 05:39:101735
mflodmancc3d4422017-08-03 15:27:511736 video_stream_encoder_->Stop();
perkj26105b42016-09-30 05:39:101737}
1738
mflodmancc3d4422017-08-03 15:27:511739TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 16:49:071740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071741 DataRate::BitsPerSec(kTargetBitrateBps),
1742 DataRate::BitsPerSec(kTargetBitrateBps),
1743 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-03 06:45:261744
1745 // Capture a frame and wait for it to synchronize with the encoder thread.
1746 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521747 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571748 // The encoder will have been configured once.
1749 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261750 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1751 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1752
1753 codec_width_ *= 2;
1754 codec_height_ *= 2;
1755 // Capture a frame with a higher resolution and wait for it to synchronize
1756 // with the encoder thread.
1757 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521758 WaitForEncodedFrame(2);
perkjfa10b552016-10-03 06:45:261759 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1760 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 20:37:571761 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261762
mflodmancc3d4422017-08-03 15:27:511763 video_stream_encoder_->Stop();
perkjfa10b552016-10-03 06:45:261764}
1765
Sergey Silkin443b7ee2019-06-28 10:53:071766TEST_F(VideoStreamEncoderTest,
1767 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 16:49:071768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071769 DataRate::BitsPerSec(kTargetBitrateBps),
1770 DataRate::BitsPerSec(kTargetBitrateBps),
1771 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 10:53:071772
1773 // Capture a frame and wait for it to synchronize with the encoder thread.
1774 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1775 WaitForEncodedFrame(1);
1776
1777 VideoEncoderConfig video_encoder_config;
1778 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1779 // Changing the max payload data length recreates encoder.
1780 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1781 kMaxPayloadLength / 2);
1782
1783 // Capture a frame and wait for it to synchronize with the encoder thread.
1784 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1785 WaitForEncodedFrame(2);
1786 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1787
1788 video_stream_encoder_->Stop();
1789}
1790
Sergey Silkin5ee69672019-07-02 12:18:341791TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 16:49:071792 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071793 DataRate::BitsPerSec(kTargetBitrateBps),
1794 DataRate::BitsPerSec(kTargetBitrateBps),
1795 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 12:18:341796
1797 VideoEncoderConfig video_encoder_config;
1798 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1799 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1800 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1801 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1802 kMaxPayloadLength);
1803
1804 // Capture a frame and wait for it to synchronize with the encoder thread.
1805 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1806 WaitForEncodedFrame(1);
1807 // The encoder will have been configured once when the first frame is
1808 // received.
1809 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1810 EXPECT_EQ(kTargetBitrateBps,
1811 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1812 EXPECT_EQ(kStartBitrateBps,
1813 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1814
Sergey Silkin6456e352019-07-08 15:56:401815 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1816 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 12:18:341817 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1818 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1819 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1820 kMaxPayloadLength);
1821
1822 // Capture a frame and wait for it to synchronize with the encoder thread.
1823 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1824 WaitForEncodedFrame(2);
1825 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1826 // Bitrate limits have changed - rate allocator should be reconfigured,
1827 // encoder should not be reconfigured.
1828 EXPECT_EQ(kTargetBitrateBps * 2,
1829 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1830 EXPECT_EQ(kStartBitrateBps * 2,
1831 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1832 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1833
1834 video_stream_encoder_->Stop();
1835}
1836
Sergey Silkin6456e352019-07-08 15:56:401837TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 13:48:401838 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 16:49:071839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071840 DataRate::BitsPerSec(kTargetBitrateBps),
1841 DataRate::BitsPerSec(kTargetBitrateBps),
1842 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401843
Sergey Silkincd02eba2020-01-20 13:48:401844 const uint32_t kMinEncBitrateKbps = 100;
1845 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051846 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 13:48:401847 /*frame_size_pixels=*/codec_width_ * codec_height_,
1848 /*min_start_bitrate_bps=*/0,
1849 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1850 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051851 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1852
Sergey Silkincd02eba2020-01-20 13:48:401853 VideoEncoderConfig video_encoder_config;
1854 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1855 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1856 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1857 (kMinEncBitrateKbps + 1) * 1000;
1858 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1859 kMaxPayloadLength);
1860
1861 // When both encoder and app provide bitrate limits, the intersection of
1862 // provided sets should be used.
1863 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1864 WaitForEncodedFrame(1);
1865 EXPECT_EQ(kMaxEncBitrateKbps,
1866 bitrate_allocator_factory_.codec_config().maxBitrate);
1867 EXPECT_EQ(kMinEncBitrateKbps + 1,
1868 bitrate_allocator_factory_.codec_config().minBitrate);
1869
1870 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1871 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1872 (kMinEncBitrateKbps - 1) * 1000;
1873 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1874 kMaxPayloadLength);
1875 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 14:04:051876 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 13:48:401877 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 14:04:051878 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:401879 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051880 bitrate_allocator_factory_.codec_config().minBitrate);
1881
Sergey Silkincd02eba2020-01-20 13:48:401882 video_stream_encoder_->Stop();
1883}
1884
1885TEST_F(VideoStreamEncoderTest,
1886 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 16:49:071887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071888 DataRate::BitsPerSec(kTargetBitrateBps),
1889 DataRate::BitsPerSec(kTargetBitrateBps),
1890 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 13:48:401891
1892 const uint32_t kMinAppBitrateKbps = 100;
1893 const uint32_t kMaxAppBitrateKbps = 200;
1894 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1895 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1896 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1897 /*frame_size_pixels=*/codec_width_ * codec_height_,
1898 /*min_start_bitrate_bps=*/0,
1899 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1900 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1901 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1902
1903 VideoEncoderConfig video_encoder_config;
1904 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1905 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1906 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1907 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051908 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1909 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 14:04:051910
Sergey Silkincd02eba2020-01-20 13:48:401911 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1912 WaitForEncodedFrame(1);
1913 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051914 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:401915 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051916 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 15:56:401917
1918 video_stream_encoder_->Stop();
1919}
1920
1921TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 14:04:051922 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 16:49:071923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071924 DataRate::BitsPerSec(kTargetBitrateBps),
1925 DataRate::BitsPerSec(kTargetBitrateBps),
1926 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401927
1928 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 14:04:051929 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 15:56:401930 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 14:04:051931 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 15:56:401932 fake_encoder_.SetResolutionBitrateLimits(
1933 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1934
1935 VideoEncoderConfig video_encoder_config;
1936 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1937 video_encoder_config.max_bitrate_bps = 0;
1938 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1939 kMaxPayloadLength);
1940
Sergey Silkin6b2cec12019-08-09 14:04:051941 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 15:56:401942 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1943 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 14:04:051944 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1945 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401946 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1947 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1948
Sergey Silkin6b2cec12019-08-09 14:04:051949 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 15:56:401950 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1951 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 14:04:051952 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1953 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401954 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1955 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1956
Sergey Silkin6b2cec12019-08-09 14:04:051957 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 15:56:401958 // encoder for 360p should be used.
1959 video_source_.IncomingCapturedFrame(
1960 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1961 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 14:04:051962 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1963 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401964 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1965 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1966
Sergey Silkin6b2cec12019-08-09 14:04:051967 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 15:56:401968 // ignored.
1969 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1970 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 14:04:051971 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1972 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401973 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1974 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051975 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1976 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401977 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1978 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1979
1980 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1981 // for 270p should be used.
1982 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1983 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 14:04:051984 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1985 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401986 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1987 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1988
1989 video_stream_encoder_->Stop();
1990}
1991
Sergey Silkin6b2cec12019-08-09 14:04:051992TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 16:49:071993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:071994 DataRate::BitsPerSec(kTargetBitrateBps),
1995 DataRate::BitsPerSec(kTargetBitrateBps),
1996 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 14:04:051997
1998 VideoEncoderConfig video_encoder_config;
1999 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2000 video_encoder_config.max_bitrate_bps = 0;
2001 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2002 kMaxPayloadLength);
2003
2004 // Encode 720p frame to get the default encoder target bitrate.
2005 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2006 WaitForEncodedFrame(1);
2007 const uint32_t kDefaultTargetBitrateFor720pKbps =
2008 bitrate_allocator_factory_.codec_config()
2009 .simulcastStream[0]
2010 .targetBitrate;
2011
2012 // Set the max recommended encoder bitrate to something lower than the default
2013 // target bitrate.
2014 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2015 1280 * 720, 10 * 1000, 10 * 1000,
2016 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2017 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2018
2019 // Change resolution to trigger encoder reinitialization.
2020 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2021 WaitForEncodedFrame(2);
2022 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2023 WaitForEncodedFrame(3);
2024
2025 // Ensure the target bitrate is capped by the max bitrate.
2026 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2027 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2028 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2029 .simulcastStream[0]
2030 .targetBitrate *
2031 1000,
2032 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2033
2034 video_stream_encoder_->Stop();
2035}
2036
Åsa Perssona7e34d32021-01-20 14:36:132037TEST_F(VideoStreamEncoderTest,
2038 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2039 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2040 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2041 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2042 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2043 fake_encoder_.SetResolutionBitrateLimits(
2044 {kEncoderLimits270p, kEncoderLimits360p});
2045
2046 // Two streams, highest stream active.
2047 VideoEncoderConfig config;
2048 const int kNumStreams = 2;
2049 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2050 config.max_bitrate_bps = 0;
2051 config.simulcast_layers[0].active = false;
2052 config.simulcast_layers[1].active = true;
2053 config.video_stream_factory =
2054 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2055 "VP8", /*max qp*/ 56, /*screencast*/ false,
2056 /*screenshare enabled*/ false);
2057 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2058
2059 // The encoder bitrate limits for 270p should be used.
2060 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2061 EXPECT_FALSE(WaitForFrame(1000));
2062 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2063 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2064 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2065 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2066 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2067
2068 // The encoder bitrate limits for 360p should be used.
2069 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2070 EXPECT_FALSE(WaitForFrame(1000));
2071 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2072 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2073 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2074 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2075
2076 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2077 video_source_.IncomingCapturedFrame(
2078 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2079 EXPECT_FALSE(WaitForFrame(1000));
2080 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2081 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2082 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2083 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2084
2085 // Resolution higher than 360p. Encoder limits should be ignored.
2086 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2087 EXPECT_FALSE(WaitForFrame(1000));
2088 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2089 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2090 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2091 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2092 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2093 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2094 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2095 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2096
2097 // Resolution lower than 270p. The encoder limits for 270p should be used.
2098 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2099 EXPECT_FALSE(WaitForFrame(1000));
2100 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2101 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2102 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2103 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2104
2105 video_stream_encoder_->Stop();
2106}
2107
2108TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:512109 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2110 // Two streams, highest stream active.
2111 VideoEncoderConfig config;
2112 const int kNumStreams = 2;
2113 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2114 config.max_bitrate_bps = 0;
2115 config.simulcast_layers[0].active = false;
2116 config.simulcast_layers[1].active = true;
2117 config.video_stream_factory =
2118 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2119 "VP8", /*max qp*/ 56, /*screencast*/ false,
2120 /*screenshare enabled*/ false);
2121 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2122
2123 // Default bitrate limits for 270p should be used.
2124 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2125 kDefaultLimits270p =
2126 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192127 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:512128 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2129 EXPECT_FALSE(WaitForFrame(1000));
2130 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2131 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2132 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2133 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2134 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2135
2136 // Default bitrate limits for 360p should be used.
2137 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2138 kDefaultLimits360p =
2139 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192140 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:512141 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2142 EXPECT_FALSE(WaitForFrame(1000));
2143 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2144 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2145 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2146 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2147
2148 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2149 video_source_.IncomingCapturedFrame(
2150 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2151 EXPECT_FALSE(WaitForFrame(1000));
2152 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2153 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2154 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2155 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2156
2157 // Default bitrate limits for 540p should be used.
2158 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2159 kDefaultLimits540p =
2160 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 12:29:192161 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 09:39:512162 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2163 EXPECT_FALSE(WaitForFrame(1000));
2164 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2165 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2166 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2167 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2168
2169 video_stream_encoder_->Stop();
2170}
2171
2172TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:132173 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2174 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2175 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2176 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2177 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2178 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2179 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2180 fake_encoder_.SetResolutionBitrateLimits(
2181 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2182
2183 // Three streams, middle stream active.
2184 VideoEncoderConfig config;
2185 const int kNumStreams = 3;
2186 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2187 config.simulcast_layers[0].active = false;
2188 config.simulcast_layers[1].active = true;
2189 config.simulcast_layers[2].active = false;
2190 config.video_stream_factory =
2191 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2192 "VP8", /*max qp*/ 56, /*screencast*/ false,
2193 /*screenshare enabled*/ false);
2194 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2195
2196 // The encoder bitrate limits for 360p should be used.
2197 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2198 EXPECT_FALSE(WaitForFrame(1000));
2199 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2200 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2201 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2202 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2203 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2204
2205 // The encoder bitrate limits for 270p should be used.
2206 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2207 EXPECT_FALSE(WaitForFrame(1000));
2208 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2209 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2210 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2211 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2212
2213 video_stream_encoder_->Stop();
2214}
2215
2216TEST_F(VideoStreamEncoderTest,
2217 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2218 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2219 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2220 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2221 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2222 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2223 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2224 fake_encoder_.SetResolutionBitrateLimits(
2225 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2226
2227 // Three streams, lowest stream active.
2228 VideoEncoderConfig config;
2229 const int kNumStreams = 3;
2230 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2231 config.simulcast_layers[0].active = true;
2232 config.simulcast_layers[1].active = false;
2233 config.simulcast_layers[2].active = false;
2234 config.video_stream_factory =
2235 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2236 "VP8", /*max qp*/ 56, /*screencast*/ false,
2237 /*screenshare enabled*/ false);
2238 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2239
2240 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2241 // on lowest stream, limits for 270p should not be used
2242 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2243 EXPECT_FALSE(WaitForFrame(1000));
2244 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2245 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2246 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2247 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2248 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2249
2250 video_stream_encoder_->Stop();
2251}
2252
2253TEST_F(VideoStreamEncoderTest,
2254 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2255 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2256 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2257 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2258 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2259 fake_encoder_.SetResolutionBitrateLimits(
2260 {kEncoderLimits270p, kEncoderLimits360p});
2261 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2262
2263 // Two streams, highest stream active.
2264 VideoEncoderConfig config;
2265 const int kNumStreams = 2;
2266 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2267 config.simulcast_layers[0].active = false;
2268 config.simulcast_layers[1].active = true;
2269 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2270 config.video_stream_factory =
2271 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2272 "VP8", /*max qp*/ 56, /*screencast*/ false,
2273 /*screenshare enabled*/ false);
2274 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2275
2276 // The encoder bitrate limits for 270p should be used.
2277 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2278 EXPECT_FALSE(WaitForFrame(1000));
2279 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2280 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2281 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2282 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2283 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2284
2285 // The max configured bitrate is less than the encoder limit for 360p.
2286 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2287 EXPECT_FALSE(WaitForFrame(1000));
2288 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2289 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2290 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2291 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2292
2293 video_stream_encoder_->Stop();
2294}
2295
mflodmancc3d4422017-08-03 15:27:512296TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 18:45:462297 EXPECT_TRUE(video_source_.has_sinks());
2298 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512299 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412300 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:462301 EXPECT_FALSE(video_source_.has_sinks());
2302 EXPECT_TRUE(new_video_source.has_sinks());
2303
mflodmancc3d4422017-08-03 15:27:512304 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462305}
2306
mflodmancc3d4422017-08-03 15:27:512307TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 18:45:462308 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512309 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 18:45:462310 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:512311 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462312}
2313
Åsa Perssonc5a74ff2020-09-20 15:50:002314class ResolutionAlignmentTest
2315 : public VideoStreamEncoderTest,
2316 public ::testing::WithParamInterface<
2317 ::testing::tuple<int, std::vector<double>>> {
2318 public:
2319 ResolutionAlignmentTest()
2320 : requested_alignment_(::testing::get<0>(GetParam())),
2321 scale_factors_(::testing::get<1>(GetParam())) {}
2322
2323 protected:
2324 const int requested_alignment_;
2325 const std::vector<double> scale_factors_;
2326};
2327
2328INSTANTIATE_TEST_SUITE_P(
2329 AlignmentAndScaleFactors,
2330 ResolutionAlignmentTest,
2331 ::testing::Combine(
2332 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2333 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2334 std::vector<double>{-1.0, -1.0},
2335 std::vector<double>{-1.0, -1.0, -1.0},
2336 std::vector<double>{4.0, 2.0, 1.0},
2337 std::vector<double>{9999.0, -1.0, 1.0},
2338 std::vector<double>{3.99, 2.01, 1.0},
2339 std::vector<double>{4.9, 1.7, 1.25},
2340 std::vector<double>{10.0, 4.0, 3.0},
2341 std::vector<double>{1.75, 3.5},
2342 std::vector<double>{1.5, 2.5},
2343 std::vector<double>{1.3, 1.0})));
2344
2345TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2346 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 08:47:112347 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 15:50:002348 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2349 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2350
2351 // Fill config with the scaling factor by which to reduce encoding size.
2352 const int num_streams = scale_factors_.size();
2353 VideoEncoderConfig config;
2354 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2355 for (int i = 0; i < num_streams; ++i) {
2356 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2357 }
2358 config.video_stream_factory =
2359 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2360 "VP8", /*max qp*/ 56, /*screencast*/ false,
2361 /*screenshare enabled*/ false);
2362 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2363
Henrik Boström381d1092020-05-12 16:49:072364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 15:50:002365 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2366 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2367 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2368 // Wait for all layers before triggering event.
2369 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 08:47:112370
2371 // On the 1st frame, we should have initialized the encoder and
2372 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 15:50:002373 int64_t timestamp_ms = kFrameIntervalMs;
2374 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2375 WaitForEncodedFrame(timestamp_ms);
2376 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 08:47:112377
2378 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2379 // (It's up the to the encoder to potentially drop the previous frame,
2380 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 15:50:002381 timestamp_ms += kFrameIntervalMs;
2382 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2383 WaitForEncodedFrame(timestamp_ms);
2384 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2385
2386 VideoCodec codec = fake_encoder_.video_codec();
2387 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2388 // Frame size should be a multiple of the requested alignment.
2389 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2390 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2391 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2392 // Aspect ratio should match.
2393 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2394 codec.height * codec.simulcastStream[i].width);
2395 }
Rasmus Brandt5cad55b2019-12-19 08:47:112396
2397 video_stream_encoder_->Stop();
2398}
2399
Jonathan Yubc771b72017-12-09 01:04:292400TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2401 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-14 06:25:222402 const int kWidth = 1280;
2403 const int kHeight = 720;
Jonathan Yubc771b72017-12-09 01:04:292404
2405 // We rely on the automatic resolution adaptation, but we handle framerate
2406 // adaptation manually by mocking the stats proxy.
2407 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-14 06:25:222408
Taylor Brandstetter49fcc102018-05-16 21:20:412409 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 16:49:072410 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072411 DataRate::BitsPerSec(kTargetBitrateBps),
2412 DataRate::BitsPerSec(kTargetBitrateBps),
2413 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 21:20:412414 video_stream_encoder_->SetSource(&video_source_,
2415 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:392416 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:222417 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292418 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:222419 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2420
Jonathan Yubc771b72017-12-09 01:04:292421 // Adapt down as far as possible.
2422 rtc::VideoSinkWants last_wants;
2423 int64_t t = 1;
2424 int loop_count = 0;
2425 do {
2426 ++loop_count;
2427 last_wants = video_source_.sink_wants();
2428
2429 // Simulate the framerate we've been asked to adapt to.
2430 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2431 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2432 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2433 mock_stats.input_frame_rate = fps;
2434 stats_proxy_->SetMockStats(mock_stats);
2435
2436 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2437 sink_.WaitForEncodedFrame(t);
2438 t += frame_interval_ms;
2439
mflodmancc3d4422017-08-03 15:27:512440 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392441 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292442 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392443 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2444 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292445 } while (video_source_.sink_wants().max_pixel_count <
2446 last_wants.max_pixel_count ||
2447 video_source_.sink_wants().max_framerate_fps <
2448 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222449
Jonathan Yubc771b72017-12-09 01:04:292450 // Verify that we've adapted all the way down.
2451 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222452 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292453 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2454 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-14 06:25:222455 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:292456 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2457 *video_source_.last_sent_height());
2458 EXPECT_EQ(kMinBalancedFramerateFps,
2459 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222460
Jonathan Yubc771b72017-12-09 01:04:292461 // Adapt back up the same number of times we adapted down.
2462 for (int i = 0; i < loop_count - 1; ++i) {
2463 last_wants = video_source_.sink_wants();
2464
2465 // Simulate the framerate we've been asked to adapt to.
2466 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2467 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2468 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2469 mock_stats.input_frame_rate = fps;
2470 stats_proxy_->SetMockStats(mock_stats);
2471
2472 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2473 sink_.WaitForEncodedFrame(t);
2474 t += frame_interval_ms;
2475
Henrik Boström91aa7322020-04-28 10:24:332476 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:392477 EXPECT_THAT(
Jonathan Yubc771b72017-12-09 01:04:292478 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 12:08:392479 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2480 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-09 01:04:292481 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2482 last_wants.max_pixel_count ||
2483 video_source_.sink_wants().max_framerate_fps >
2484 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:222485 }
2486
Evan Shrubsole5cd7eb82020-05-25 12:08:392487 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-09 01:04:292488 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:222489 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:292490 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2491 EXPECT_EQ((loop_count - 1) * 2,
2492 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:222493
mflodmancc3d4422017-08-03 15:27:512494 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222495}
Rasmus Brandt5cad55b2019-12-19 08:47:112496
Evan Shrubsole2e2f6742020-05-14 08:41:152497TEST_F(VideoStreamEncoderTest,
2498 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2499 video_stream_encoder_->OnBitrateUpdated(
2500 DataRate::BitsPerSec(kTargetBitrateBps),
2501 DataRate::BitsPerSec(kTargetBitrateBps),
2502 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392503 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152504
2505 const int kFrameWidth = 1280;
2506 const int kFrameHeight = 720;
2507
2508 int64_t ntp_time = kFrameIntervalMs;
2509
2510 // Force an input frame rate to be available, or the adaptation call won't
2511 // know what framerate to adapt form.
2512 const int kInputFps = 30;
2513 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2514 stats.input_frame_rate = kInputFps;
2515 stats_proxy_->SetMockStats(stats);
2516
2517 video_source_.set_adaptation_enabled(true);
2518 video_stream_encoder_->SetSource(
2519 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 12:08:392520 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 08:41:152521 video_source_.IncomingCapturedFrame(
2522 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2523 sink_.WaitForEncodedFrame(ntp_time);
2524 ntp_time += kFrameIntervalMs;
2525
2526 // Trigger CPU overuse.
2527 video_stream_encoder_->TriggerCpuOveruse();
2528 video_source_.IncomingCapturedFrame(
2529 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2530 sink_.WaitForEncodedFrame(ntp_time);
2531 ntp_time += kFrameIntervalMs;
2532
2533 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2534 EXPECT_EQ(std::numeric_limits<int>::max(),
2535 video_source_.sink_wants().max_pixel_count);
2536 // Some framerate constraint should be set.
2537 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2538 EXPECT_LT(restricted_fps, kInputFps);
2539 video_source_.IncomingCapturedFrame(
2540 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2541 sink_.WaitForEncodedFrame(ntp_time);
2542 ntp_time += 100;
2543
Henrik Boström2671dac2020-05-19 14:29:092544 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152545 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2546 // Give the encoder queue time to process the change in degradation preference
2547 // by waiting for an encoded frame.
2548 video_source_.IncomingCapturedFrame(
2549 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2550 sink_.WaitForEncodedFrame(ntp_time);
2551 ntp_time += kFrameIntervalMs;
2552
2553 video_stream_encoder_->TriggerQualityLow();
2554 video_source_.IncomingCapturedFrame(
2555 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2556 sink_.WaitForEncodedFrame(ntp_time);
2557 ntp_time += kFrameIntervalMs;
2558
2559 // Some resolution constraint should be set.
2560 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2561 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2562 kFrameWidth * kFrameHeight);
2563 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2564
2565 int pixel_count = video_source_.sink_wants().max_pixel_count;
2566 // Triggering a CPU underuse should not change the sink wants since it has
2567 // not been overused for resolution since we changed degradation preference.
2568 video_stream_encoder_->TriggerCpuUnderuse();
2569 video_source_.IncomingCapturedFrame(
2570 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2571 sink_.WaitForEncodedFrame(ntp_time);
2572 ntp_time += kFrameIntervalMs;
2573 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2574 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2575
Evan Shrubsole64469032020-06-11 08:45:292576 // Change the degradation preference back. CPU underuse should not adapt since
2577 // QP is most limited.
Henrik Boström2671dac2020-05-19 14:29:092578 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 08:41:152579 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2580 video_source_.IncomingCapturedFrame(
2581 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2582 sink_.WaitForEncodedFrame(ntp_time);
2583 ntp_time += 100;
2584 // Resolution adaptations is gone after changing degradation preference.
2585 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2586 EXPECT_EQ(std::numeric_limits<int>::max(),
2587 video_source_.sink_wants().max_pixel_count);
2588 // The fps adaptation from above is now back.
2589 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2590
2591 // Trigger CPU underuse.
2592 video_stream_encoder_->TriggerCpuUnderuse();
2593 video_source_.IncomingCapturedFrame(
2594 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2595 sink_.WaitForEncodedFrame(ntp_time);
2596 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 08:45:292597 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2598
2599 // Trigger QP underuse, fps should return to normal.
2600 video_stream_encoder_->TriggerQualityHigh();
2601 video_source_.IncomingCapturedFrame(
2602 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2603 sink_.WaitForEncodedFrame(ntp_time);
2604 ntp_time += kFrameIntervalMs;
2605 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 08:41:152606
2607 video_stream_encoder_->Stop();
2608}
2609
mflodmancc3d4422017-08-03 15:27:512610TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 16:49:072611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072612 DataRate::BitsPerSec(kTargetBitrateBps),
2613 DataRate::BitsPerSec(kTargetBitrateBps),
2614 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 12:08:392615 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 18:45:462616
sprangc5d62e22017-04-03 06:53:042617 const int kFrameWidth = 1280;
2618 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:042619
Åsa Persson8c1bf952018-09-13 08:42:192620 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 18:45:462621
kthelgason5e13d412016-12-01 11:59:512622 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042623 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522624 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042625 frame_timestamp += kFrameIntervalMs;
2626
perkj803d97f2016-11-01 18:45:462627 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512628 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462629 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042630 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522631 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042632 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 14:23:482633
asapersson0944a802017-04-07 07:57:582634 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-03 06:53:042635 // wanted resolution.
2636 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2637 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2638 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 08:42:192639 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042640
2641 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 18:45:462642 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 16:49:072643 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412644 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122645 // Give the encoder queue time to process the change in degradation preference
2646 // by waiting for an encoded frame.
2647 new_video_source.IncomingCapturedFrame(
2648 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2649 sink_.WaitForEncodedFrame(frame_timestamp);
2650 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042651 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 12:08:392652 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462653
sprangc5d62e22017-04-03 06:53:042654 // Force an input frame rate to be available, or the adaptation call won't
2655 // know what framerate to adapt form.
asapersson02465b82017-04-10 08:12:522656 const int kInputFps = 30;
sprangc5d62e22017-04-03 06:53:042657 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 08:12:522658 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-03 06:53:042659 stats_proxy_->SetMockStats(stats);
2660
mflodmancc3d4422017-08-03 15:27:512661 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:462662 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:042663 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522664 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042665 frame_timestamp += kFrameIntervalMs;
2666
2667 // Some framerate constraint should be set.
sprang84a37592017-02-10 15:04:272668 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-03 06:53:042669 EXPECT_EQ(std::numeric_limits<int>::max(),
2670 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522671 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-03 06:53:042672
asapersson02465b82017-04-10 08:12:522673 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 16:49:072674 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2675 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 10:42:122676 // Give the encoder queue time to process the change in degradation preference
2677 // by waiting for an encoded frame.
2678 new_video_source.IncomingCapturedFrame(
2679 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2680 sink_.WaitForEncodedFrame(frame_timestamp);
2681 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 12:08:392682 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-03 06:53:042683
mflodmancc3d4422017-08-03 15:27:512684 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042685 new_video_source.IncomingCapturedFrame(
2686 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:522687 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:042688 frame_timestamp += kFrameIntervalMs;
2689
2690 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 12:08:392691 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 18:45:462692
2693 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072694 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412695 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 10:42:122696 // Give the encoder queue time to process the change in degradation preference
2697 // by waiting for an encoded frame.
2698 new_video_source.IncomingCapturedFrame(
2699 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2700 sink_.WaitForEncodedFrame(frame_timestamp);
2701 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042702 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2703 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 15:04:272704 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 08:42:192705 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:042706
2707 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 16:49:072708 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:412709 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122710 // Give the encoder queue time to process the change in degradation preference
2711 // by waiting for an encoded frame.
2712 new_video_source.IncomingCapturedFrame(
2713 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2714 sink_.WaitForEncodedFrame(frame_timestamp);
2715 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:042716 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2717 EXPECT_EQ(std::numeric_limits<int>::max(),
2718 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:522719 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 18:45:462720
mflodmancc3d4422017-08-03 15:27:512721 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462722}
2723
mflodmancc3d4422017-08-03 15:27:512724TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072726 DataRate::BitsPerSec(kTargetBitrateBps),
2727 DataRate::BitsPerSec(kTargetBitrateBps),
2728 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 18:45:462729
asaperssonfab67072017-04-04 12:51:492730 const int kWidth = 1280;
2731 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492732 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522733 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 12:51:492734 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2735 EXPECT_FALSE(stats.bw_limited_resolution);
2736 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2737
2738 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512739 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492740 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522741 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 12:51:492742
2743 stats = stats_proxy_->GetStats();
2744 EXPECT_TRUE(stats.bw_limited_resolution);
2745 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2746
2747 // Trigger adapt up.
mflodmancc3d4422017-08-03 15:27:512748 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:492749 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522750 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 12:51:492751
2752 stats = stats_proxy_->GetStats();
2753 EXPECT_FALSE(stats.bw_limited_resolution);
2754 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2755 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2756
mflodmancc3d4422017-08-03 15:27:512757 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 12:51:492758}
2759
mflodmancc3d4422017-08-03 15:27:512760TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 16:49:072761 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072762 DataRate::BitsPerSec(kTargetBitrateBps),
2763 DataRate::BitsPerSec(kTargetBitrateBps),
2764 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:492765
2766 const int kWidth = 1280;
2767 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492768 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522769 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 18:45:462770 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2771 EXPECT_FALSE(stats.cpu_limited_resolution);
2772 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2773
2774 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512775 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492776 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522777 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 18:45:462778
2779 stats = stats_proxy_->GetStats();
2780 EXPECT_TRUE(stats.cpu_limited_resolution);
2781 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2782
2783 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332784 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492785 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522786 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 18:45:462787
2788 stats = stats_proxy_->GetStats();
2789 EXPECT_FALSE(stats.cpu_limited_resolution);
2790 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 12:51:492791 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:462792
mflodmancc3d4422017-08-03 15:27:512793 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462794}
2795
mflodmancc3d4422017-08-03 15:27:512796TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072797 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072798 DataRate::BitsPerSec(kTargetBitrateBps),
2799 DataRate::BitsPerSec(kTargetBitrateBps),
2800 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112801
asaperssonfab67072017-04-04 12:51:492802 const int kWidth = 1280;
2803 const int kHeight = 720;
2804 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522805 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112806 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182807 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112808 EXPECT_FALSE(stats.cpu_limited_resolution);
2809 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2810
asaperssonfab67072017-04-04 12:51:492811 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512812 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492813 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522814 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112815 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182816 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112817 EXPECT_TRUE(stats.cpu_limited_resolution);
2818 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2819
2820 // Set new source with adaptation still enabled.
2821 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512822 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412823 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112824
asaperssonfab67072017-04-04 12:51:492825 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522826 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112827 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182828 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112829 EXPECT_TRUE(stats.cpu_limited_resolution);
2830 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2831
2832 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412833 video_stream_encoder_->SetSource(&new_video_source,
2834 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 09:44:112835
asaperssonfab67072017-04-04 12:51:492836 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522837 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112838 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182839 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112840 EXPECT_FALSE(stats.cpu_limited_resolution);
2841 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2842
2843 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 15:27:512844 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412845 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112846
asaperssonfab67072017-04-04 12:51:492847 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522848 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112849 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182850 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112851 EXPECT_TRUE(stats.cpu_limited_resolution);
2852 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2853
asaperssonfab67072017-04-04 12:51:492854 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 10:24:332855 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:492856 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522857 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 09:44:112858 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182859 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112860 EXPECT_FALSE(stats.cpu_limited_resolution);
2861 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:522862 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112863
mflodmancc3d4422017-08-03 15:27:512864 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112865}
2866
mflodmancc3d4422017-08-03 15:27:512867TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 16:49:072868 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072869 DataRate::BitsPerSec(kTargetBitrateBps),
2870 DataRate::BitsPerSec(kTargetBitrateBps),
2871 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112872
asaperssonfab67072017-04-04 12:51:492873 const int kWidth = 1280;
2874 const int kHeight = 720;
2875 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522876 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112877 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112878 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022879 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492880 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112881
2882 // Set new source with adaptation still enabled.
2883 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 21:20:412884 video_stream_encoder_->SetSource(&new_video_source,
2885 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112886
asaperssonfab67072017-04-04 12:51:492887 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522888 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112889 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112890 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022891 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492892 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112893
asaperssonfab67072017-04-04 12:51:492894 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512895 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492896 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522897 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112898 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112899 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022900 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492901 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112902
asaperssonfab67072017-04-04 12:51:492903 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412904 video_stream_encoder_->SetSource(&new_video_source,
2905 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112906
asaperssonfab67072017-04-04 12:51:492907 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522908 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112909 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112910 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022911 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492912 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112913
asapersson02465b82017-04-10 08:12:522914 // Disable resolution scaling.
mflodmancc3d4422017-08-03 15:27:512915 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412916 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:112917
asaperssonfab67072017-04-04 12:51:492918 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522919 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112920 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112921 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022922 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492923 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2924 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 09:44:112925
mflodmancc3d4422017-08-03 15:27:512926 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112927}
2928
mflodmancc3d4422017-08-03 15:27:512929TEST_F(VideoStreamEncoderTest,
2930 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 16:49:072931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:072932 DataRate::BitsPerSec(kTargetBitrateBps),
2933 DataRate::BitsPerSec(kTargetBitrateBps),
2934 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 12:29:122935
2936 const int kWidth = 1280;
2937 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 08:42:192938 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 12:29:122939 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 08:42:192940 video_source_.IncomingCapturedFrame(
2941 CreateFrame(timestamp_ms, kWidth, kHeight));
2942 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122943 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2944 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2945 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2946
2947 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512948 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:192949 timestamp_ms += kFrameIntervalMs;
2950 video_source_.IncomingCapturedFrame(
2951 CreateFrame(timestamp_ms, kWidth, kHeight));
2952 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122953 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2954 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2955 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2956
2957 // Trigger overuse.
mflodmancc3d4422017-08-03 15:27:512958 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:192959 timestamp_ms += kFrameIntervalMs;
2960 video_source_.IncomingCapturedFrame(
2961 CreateFrame(timestamp_ms, kWidth, kHeight));
2962 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122963 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2964 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2965 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2966
Niels Möller4db138e2018-04-19 07:04:132967 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 12:29:122968 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:132969
2970 VideoEncoderConfig video_encoder_config;
2971 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2972 // Make format different, to force recreation of encoder.
2973 video_encoder_config.video_format.parameters["foo"] = "foo";
2974 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:472975 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 08:42:192976 timestamp_ms += kFrameIntervalMs;
2977 video_source_.IncomingCapturedFrame(
2978 CreateFrame(timestamp_ms, kWidth, kHeight));
2979 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122980 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2981 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2982 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2983
mflodmancc3d4422017-08-03 15:27:512984 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 12:29:122985}
2986
mflodmancc3d4422017-08-03 15:27:512987TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 17:39:322988 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 16:49:072989 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 17:39:322990 DataRate::BitsPerSec(kTargetBitrateBps),
2991 DataRate::BitsPerSec(kTargetBitrateBps),
2992 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2993
2994 const int kWidth = 1280;
2995 const int kHeight = 720;
2996 int sequence = 1;
2997
2998 // Enable BALANCED preference, no initial limitation.
2999 test::FrameForwarder source;
3000 video_stream_encoder_->SetSource(&source,
3001 webrtc::DegradationPreference::BALANCED);
3002 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3003 WaitForEncodedFrame(sequence++);
3004 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3005 EXPECT_FALSE(stats.cpu_limited_resolution);
3006 EXPECT_FALSE(stats.cpu_limited_framerate);
3007 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3008
3009 // Trigger CPU overuse, should now adapt down.
3010 video_stream_encoder_->TriggerCpuOveruse();
3011 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3012 WaitForEncodedFrame(sequence++);
3013 stats = stats_proxy_->GetStats();
3014 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3015
3016 // Set new degradation preference should clear restrictions since we changed
3017 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 14:54:213018 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 17:39:323019 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3020 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3021 WaitForEncodedFrame(sequence++);
3022 stats = stats_proxy_->GetStats();
3023 EXPECT_FALSE(stats.cpu_limited_resolution);
3024 EXPECT_FALSE(stats.cpu_limited_framerate);
3025 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3026
3027 // Force an input frame rate to be available, or the adaptation call won't
3028 // know what framerate to adapt from.
3029 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3030 mock_stats.input_frame_rate = 30;
3031 stats_proxy_->SetMockStats(mock_stats);
3032 video_stream_encoder_->TriggerCpuOveruse();
3033 stats_proxy_->ResetMockStats();
3034 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3035 WaitForEncodedFrame(sequence++);
3036
3037 // We have now adapted once.
3038 stats = stats_proxy_->GetStats();
3039 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3040
3041 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 14:54:213042 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3043 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 17:39:323044 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3045 WaitForEncodedFrame(sequence++);
3046 stats = stats_proxy_->GetStats();
3047 EXPECT_FALSE(stats.cpu_limited_resolution);
3048 EXPECT_FALSE(stats.cpu_limited_framerate);
3049 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3050
3051 video_stream_encoder_->Stop();
3052}
3053
3054TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:513055 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 16:49:073056 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073057 DataRate::BitsPerSec(kTargetBitrateBps),
3058 DataRate::BitsPerSec(kTargetBitrateBps),
3059 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 18:45:463060
asapersson0944a802017-04-07 07:57:583061 const int kWidth = 1280;
3062 const int kHeight = 720;
sprang84a37592017-02-10 15:04:273063 int sequence = 1;
perkj803d97f2016-11-01 18:45:463064
asaperssonfab67072017-04-04 12:51:493065 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523066 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463067 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 15:04:273068 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023069 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 15:04:273070 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3071
asapersson02465b82017-04-10 08:12:523072 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 15:27:513073 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:493074 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523075 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 15:04:273076 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 18:45:463077 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023078 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463079 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3080
3081 // Set new source with adaptation still enabled.
3082 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513083 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413084 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:463085
3086 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493087 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523088 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463089 stats = stats_proxy_->GetStats();
3090 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183091 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463092 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3093
sprangc5d62e22017-04-03 06:53:043094 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 15:27:513095 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413096 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 18:45:463097 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493098 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523099 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463100 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-03 06:53:043101 // Not adapted at first.
perkj803d97f2016-11-01 18:45:463102 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:183103 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:463104 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3105
sprangc5d62e22017-04-03 06:53:043106 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-16 06:40:183107 // know what framerate to adapt from.
sprangc5d62e22017-04-03 06:53:043108 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3109 mock_stats.input_frame_rate = 30;
3110 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513111 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043112 stats_proxy_->ResetMockStats();
3113
3114 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493115 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523116 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043117
3118 // Framerate now adapted.
3119 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:183120 EXPECT_FALSE(stats.cpu_limited_resolution);
3121 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043122 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3123
3124 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 21:20:413125 video_stream_encoder_->SetSource(&new_video_source,
3126 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-03 06:53:043127 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493128 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523129 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043130
3131 stats = stats_proxy_->GetStats();
3132 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023133 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043134 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3135
3136 // Try to trigger overuse. Should not succeed.
3137 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:513138 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043139 stats_proxy_->ResetMockStats();
3140
3141 stats = stats_proxy_->GetStats();
3142 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023143 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043144 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3145
3146 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 15:27:513147 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413148 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 12:51:493149 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523150 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463151 stats = stats_proxy_->GetStats();
3152 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023153 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043154 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 18:45:463155
3156 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333157 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 12:51:493158 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523159 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:463160 stats = stats_proxy_->GetStats();
3161 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023162 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043163 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3164
3165 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 15:27:513166 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413167 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:043168 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493169 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523170 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043171 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 07:01:023172 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-03 06:53:043173 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023174 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043175 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3176
3177 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 10:24:333178 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:043179 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493180 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523181 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:043182 stats = stats_proxy_->GetStats();
3183 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:023184 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:043185 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:523186 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:463187
mflodmancc3d4422017-08-03 15:27:513188 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:463189}
3190
mflodmancc3d4422017-08-03 15:27:513191TEST_F(VideoStreamEncoderTest,
3192 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 12:51:493193 const int kWidth = 1280;
3194 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073196 DataRate::BitsPerSec(kTargetBitrateBps),
3197 DataRate::BitsPerSec(kTargetBitrateBps),
3198 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:113199
asaperssonfab67072017-04-04 12:51:493200 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 12:08:393201 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 09:44:113202
asaperssonfab67072017-04-04 12:51:493203 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523204 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:113205
asaperssonfab67072017-04-04 12:51:493206 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513207 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 11:59:513208
asaperssonfab67072017-04-04 12:51:493209 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523210 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 11:59:513211
kthelgason876222f2016-11-29 09:44:113212 // Expect a scale down.
3213 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 12:51:493214 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 09:44:113215
asapersson02465b82017-04-10 08:12:523216 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 09:44:113217 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:513218 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413219 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:113220
asaperssonfab67072017-04-04 12:51:493221 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:513222 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:493223 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523224 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:113225
asaperssonfab67072017-04-04 12:51:493226 // Expect no scaling.
sprangc5d62e22017-04-03 06:53:043227 EXPECT_EQ(std::numeric_limits<int>::max(),
3228 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113229
asaperssonfab67072017-04-04 12:51:493230 // Trigger scale up.
mflodmancc3d4422017-08-03 15:27:513231 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:493232 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523233 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:113234
asapersson02465b82017-04-10 08:12:523235 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-03 06:53:043236 EXPECT_EQ(std::numeric_limits<int>::max(),
3237 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:113238
mflodmancc3d4422017-08-03 15:27:513239 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:113240}
3241
mflodmancc3d4422017-08-03 15:27:513242TEST_F(VideoStreamEncoderTest,
3243 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523244 const int kWidth = 1280;
3245 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073246 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073247 DataRate::BitsPerSec(kTargetBitrateBps),
3248 DataRate::BitsPerSec(kTargetBitrateBps),
3249 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:523250
Taylor Brandstetter49fcc102018-05-16 21:20:413251 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523252 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513253 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413254 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523255
3256 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523257 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393258 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523259 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3260 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3261
3262 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513263 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393264 EXPECT_THAT(source.sink_wants(),
3265 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523266 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3267 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3268 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3269
3270 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 15:27:513271 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 08:12:523272 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3273 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3274 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3275
mflodmancc3d4422017-08-03 15:27:513276 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523277}
3278
mflodmancc3d4422017-08-03 15:27:513279TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223280 const int kWidth = 1280;
3281 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073282 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073283 DataRate::BitsPerSec(kTargetBitrateBps),
3284 DataRate::BitsPerSec(kTargetBitrateBps),
3285 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223286
Taylor Brandstetter49fcc102018-05-16 21:20:413287 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223288 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413289 video_stream_encoder_->SetSource(&source,
3290 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223291 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3292 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393293 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223294
3295 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513296 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:393297 EXPECT_THAT(source.sink_wants(),
3298 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223299 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3300 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3301 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3302
3303 // Trigger adapt down for same input resolution, expect no change.
3304 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3305 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:513306 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223307 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3308 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3309 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3310
3311 // Trigger adapt down for larger input resolution, expect no change.
3312 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3313 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:513314 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223315 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3316 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3317 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3318
mflodmancc3d4422017-08-03 15:27:513319 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223320}
3321
mflodmancc3d4422017-08-03 15:27:513322TEST_F(VideoStreamEncoderTest,
3323 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523324 const int kWidth = 1280;
3325 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073327 DataRate::BitsPerSec(kTargetBitrateBps),
3328 DataRate::BitsPerSec(kTargetBitrateBps),
3329 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:523330
Taylor Brandstetter49fcc102018-05-16 21:20:413331 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523332 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513333 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413334 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523335
3336 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523337 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393338 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523339 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3340 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3341
3342 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333343 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393344 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523345 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3346 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3347
mflodmancc3d4422017-08-03 15:27:513348 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523349}
3350
mflodmancc3d4422017-08-03 15:27:513351TEST_F(VideoStreamEncoderTest,
3352 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 08:12:523353 const int kWidth = 1280;
3354 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073356 DataRate::BitsPerSec(kTargetBitrateBps),
3357 DataRate::BitsPerSec(kTargetBitrateBps),
3358 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:523359
Taylor Brandstetter49fcc102018-05-16 21:20:413360 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 08:12:523361 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513362 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413363 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 08:12:523364
3365 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523366 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393367 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183368 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523369 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3370
3371 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 10:24:333372 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 12:08:393373 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183374 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:523375 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3376
mflodmancc3d4422017-08-03 15:27:513377 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523378}
3379
mflodmancc3d4422017-08-03 15:27:513380TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223381 const int kWidth = 1280;
3382 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073383 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073384 DataRate::BitsPerSec(kTargetBitrateBps),
3385 DataRate::BitsPerSec(kTargetBitrateBps),
3386 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223387
Taylor Brandstetter49fcc102018-05-16 21:20:413388 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223389 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413390 video_stream_encoder_->SetSource(&source,
3391 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223392
3393 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3394 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393395 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223396 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3397 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3398 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3399
3400 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513401 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393402 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3404 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3405 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3406
mflodmancc3d4422017-08-03 15:27:513407 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223408}
3409
mflodmancc3d4422017-08-03 15:27:513410TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-16 06:40:183411 const int kWidth = 1280;
3412 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073413 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073414 DataRate::BitsPerSec(kTargetBitrateBps),
3415 DataRate::BitsPerSec(kTargetBitrateBps),
3416 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:183417
Taylor Brandstetter49fcc102018-05-16 21:20:413418 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-16 06:40:183419 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413420 video_stream_encoder_->SetSource(&source,
3421 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-16 06:40:183422
3423 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3424 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393425 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183426 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3427 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3428 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3429
3430 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:513431 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393432 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3434 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3435 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
mflodmancc3d4422017-08-03 15:27:513437 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183438}
3439
mflodmancc3d4422017-08-03 15:27:513440TEST_F(VideoStreamEncoderTest,
3441 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:523442 const int kWidth = 1280;
3443 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073444 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073445 DataRate::BitsPerSec(kTargetBitrateBps),
3446 DataRate::BitsPerSec(kTargetBitrateBps),
3447 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:523448
Taylor Brandstetter49fcc102018-05-16 21:20:413449 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233450 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 08:12:523451 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513452 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413453 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:523454
3455 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523456 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393457 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3459 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513462 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 08:12:523463 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523464 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393465 EXPECT_THAT(source.sink_wants(),
3466 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 08:12:523467 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3468 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3469
3470 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513471 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393472 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 08:12:523473 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3474 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3475 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3476
mflodmancc3d4422017-08-03 15:27:513477 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523478}
3479
mflodmancc3d4422017-08-03 15:27:513480TEST_F(VideoStreamEncoderTest,
3481 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-16 06:40:183482 const int kWidth = 1280;
3483 const int kHeight = 720;
3484 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 16:49:073485 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073486 DataRate::BitsPerSec(kTargetBitrateBps),
3487 DataRate::BitsPerSec(kTargetBitrateBps),
3488 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:183489
3490 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3491 stats.input_frame_rate = kInputFps;
3492 stats_proxy_->SetMockStats(stats);
3493
Taylor Brandstetter49fcc102018-05-16 21:20:413494 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-16 06:40:183495 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3496 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 12:08:393497 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183498
3499 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513500 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-16 06:40:183501 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3502 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 12:08:393503 EXPECT_THAT(video_source_.sink_wants(),
3504 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-16 06:40:183505
Taylor Brandstetter49fcc102018-05-16 21:20:413506 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-16 06:40:183507 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 14:29:093508 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:413509 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:123510 // Give the encoder queue time to process the change in degradation preference
3511 // by waiting for an encoded frame.
3512 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3513 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 12:08:393514 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183515
3516 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 15:27:513517 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 10:42:123518 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3519 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 12:08:393520 EXPECT_THAT(new_video_source.sink_wants(),
3521 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-16 06:40:183522
3523 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513524 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393525 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-16 06:40:183526
mflodmancc3d4422017-08-03 15:27:513527 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:183528}
3529
mflodmancc3d4422017-08-03 15:27:513530TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 08:47:313531 const int kWidth = 1280;
3532 const int kHeight = 720;
3533 const size_t kNumFrames = 10;
3534
Henrik Boström381d1092020-05-12 16:49:073535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073536 DataRate::BitsPerSec(kTargetBitrateBps),
3537 DataRate::BitsPerSec(kTargetBitrateBps),
3538 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 11:59:513539
asaperssond0de2952017-04-21 08:47:313540 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 15:58:543541 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 08:47:313542 video_source_.set_adaptation_enabled(true);
3543
3544 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3545 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3546
3547 int downscales = 0;
3548 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 08:42:193549 video_source_.IncomingCapturedFrame(
3550 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3551 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 08:47:313552
asaperssonfab67072017-04-04 12:51:493553 // Trigger scale down.
asaperssond0de2952017-04-21 08:47:313554 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 15:27:513555 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-03 06:53:043556 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 08:47:313557
3558 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3559 ++downscales;
3560
3561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3562 EXPECT_EQ(downscales,
3563 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 11:59:513565 }
mflodmancc3d4422017-08-03 15:27:513566 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313567}
3568
mflodmancc3d4422017-08-03 15:27:513569TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:313570 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3571 const int kWidth = 1280;
3572 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073573 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073574 DataRate::BitsPerSec(kTargetBitrateBps),
3575 DataRate::BitsPerSec(kTargetBitrateBps),
3576 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:313577
Taylor Brandstetter49fcc102018-05-16 21:20:413578 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233579 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:313580 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513581 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413582 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:313583
Åsa Persson8c1bf952018-09-13 08:42:193584 int64_t timestamp_ms = kFrameIntervalMs;
3585 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523586 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393587 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313588 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3589 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3590
3591 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513592 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193593 timestamp_ms += kFrameIntervalMs;
3594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3595 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393596 EXPECT_THAT(source.sink_wants(),
3597 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313598 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3599 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3600
3601 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333602 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193603 timestamp_ms += kFrameIntervalMs;
3604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523605 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393606 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313607 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3608 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3609
3610 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513611 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193612 timestamp_ms += kFrameIntervalMs;
3613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393615 EXPECT_THAT(source.sink_wants(),
3616 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:313617 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3618 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3619
3620 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 10:24:333621 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:193622 timestamp_ms += kFrameIntervalMs;
3623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-16 06:40:183624 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393625 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:313626 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3627 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3628
mflodmancc3d4422017-08-03 15:27:513629 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313630}
3631
mflodmancc3d4422017-08-03 15:27:513632TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-14 06:25:223633 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3634 const int kWidth = 1280;
3635 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:073636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073637 DataRate::BitsPerSec(kTargetBitrateBps),
3638 DataRate::BitsPerSec(kTargetBitrateBps),
3639 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:223640
Taylor Brandstetter49fcc102018-05-16 21:20:413641 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233642 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:223643 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:413644 video_stream_encoder_->SetSource(&source,
3645 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223646
Åsa Persson8c1bf952018-09-13 08:42:193647 int64_t timestamp_ms = kFrameIntervalMs;
3648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223649 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393650 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3652 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3653
3654 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513655 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193656 timestamp_ms += kFrameIntervalMs;
3657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3658 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393659 EXPECT_THAT(source.sink_wants(),
3660 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223661 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3662 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3663
3664 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513665 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193666 timestamp_ms += kFrameIntervalMs;
3667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223668 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393669 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223670 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3671 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3672
3673 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:513674 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193675 timestamp_ms += kFrameIntervalMs;
3676 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3677 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:393678 EXPECT_THAT(source.sink_wants(),
3679 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:223680 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3681 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3682
3683 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:513684 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193685 timestamp_ms += kFrameIntervalMs;
3686 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:223687 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:393688 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:223689 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3690 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3691
mflodmancc3d4422017-08-03 15:27:513692 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223693}
3694
Sergey Silkin41c650b2019-10-14 11:12:193695TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3696 fake_encoder_.SetResolutionBitrateLimits(
3697 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3698
Henrik Boström381d1092020-05-12 16:49:073699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073700 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3701 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3702 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3703 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:193704
3705 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233706 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:193707 source.set_adaptation_enabled(true);
3708 video_stream_encoder_->SetSource(
3709 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3710
3711 // Insert 720p frame.
3712 int64_t timestamp_ms = kFrameIntervalMs;
3713 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3714 WaitForEncodedFrame(1280, 720);
3715
3716 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 16:49:073717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073718 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3719 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3720 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3721 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:193722 video_stream_encoder_->TriggerQualityLow();
3723
3724 // Insert 720p frame. It should be downscaled and encoded.
3725 timestamp_ms += kFrameIntervalMs;
3726 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3727 WaitForEncodedFrame(960, 540);
3728
3729 // Trigger adapt up. Higher resolution should not be requested duo to lack
3730 // of bitrate.
3731 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393732 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 11:12:193733
3734 // Increase bitrate.
Henrik Boström381d1092020-05-12 16:49:073735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073736 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3737 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3738 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3739 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:193740
3741 // Trigger adapt up. Higher resolution should be requested.
3742 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:393743 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 11:12:193744
3745 video_stream_encoder_->Stop();
3746}
3747
3748TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3749 fake_encoder_.SetResolutionBitrateLimits(
3750 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3751
3752 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 16:49:073753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073754 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3755 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3756 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3757 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:193758
3759 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:233760 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 11:12:193761 source.set_adaptation_enabled(true);
3762 video_stream_encoder_->SetSource(
3763 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3764
3765 // Insert 720p frame. It should be dropped and lower resolution should be
3766 // requested.
3767 int64_t timestamp_ms = kFrameIntervalMs;
3768 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3769 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:093770 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 11:12:193771
3772 // Insert 720p frame. It should be downscaled and encoded.
3773 timestamp_ms += kFrameIntervalMs;
3774 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3775 WaitForEncodedFrame(960, 540);
3776
3777 video_stream_encoder_->Stop();
3778}
3779
Åsa Perssonb67c44c2019-09-24 13:25:323780class BalancedDegradationTest : public VideoStreamEncoderTest {
3781 protected:
3782 void SetupTest() {
3783 // Reset encoder for field trials to take effect.
3784 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 13:13:043785 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 13:25:323786
3787 // Enable BALANCED preference.
3788 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 13:13:043789 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3790 }
3791
3792 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 16:49:073793 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:073794 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3795 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 13:25:323796 }
3797
Åsa Persson45b176f2019-09-30 09:19:053798 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 13:25:323799 timestamp_ms_ += kFrameIntervalMs;
3800 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 09:19:053801 }
3802
3803 void InsertFrameAndWaitForEncoded() {
3804 InsertFrame();
Åsa Perssonb67c44c2019-09-24 13:25:323805 sink_.WaitForEncodedFrame(timestamp_ms_);
3806 }
3807
3808 const int kWidth = 640; // pixels:640x360=230400
3809 const int kHeight = 360;
3810 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3811 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 12:31:233812 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 13:25:323813};
3814
Evan Shrubsolea1c77f62020-08-10 09:01:063815TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 13:25:323816 test::ScopedFieldTrials field_trials(
3817 "WebRTC-Video-BalancedDegradationSettings/"
3818 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3819 SetupTest();
3820
3821 // Force input frame rate.
3822 const int kInputFps = 24;
3823 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3824 stats.input_frame_rate = kInputFps;
3825 stats_proxy_->SetMockStats(stats);
3826
Åsa Persson45b176f2019-09-30 09:19:053827 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:393828 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:323829
Evan Shrubsolea1c77f62020-08-10 09:01:063830 // Trigger adapt down, expect scaled down framerate and resolution,
3831 // since Fps diff (input-requested:0) < threshold.
3832 video_stream_encoder_->TriggerQualityLow();
3833 EXPECT_THAT(source_.sink_wants(),
3834 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 13:25:323835
3836 video_stream_encoder_->Stop();
3837}
3838
Evan Shrubsolea1c77f62020-08-10 09:01:063839TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 13:25:323840 test::ScopedFieldTrials field_trials(
3841 "WebRTC-Video-BalancedDegradationSettings/"
3842 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3843 SetupTest();
3844
3845 // Force input frame rate.
3846 const int kInputFps = 25;
3847 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3848 stats.input_frame_rate = kInputFps;
3849 stats_proxy_->SetMockStats(stats);
3850
Åsa Persson45b176f2019-09-30 09:19:053851 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:393852 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:323853
Evan Shrubsolea1c77f62020-08-10 09:01:063854 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3855 // Fps diff (input-requested:1) == threshold.
3856 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:393857 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 13:25:323858
3859 video_stream_encoder_->Stop();
3860}
3861
3862TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3863 test::ScopedFieldTrials field_trials(
3864 "WebRTC-Video-BalancedDegradationSettings/"
3865 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3866 SetupTest();
3867
3868 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3869
Åsa Persson45b176f2019-09-30 09:19:053870 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:393871 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 13:25:323872
3873 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3874 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:393875 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 13:25:323876
3877 video_stream_encoder_->Stop();
3878}
3879
Åsa Perssonccfb3402019-09-25 13:13:043880TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:323881 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 15:26:393882 "WebRTC-Video-BalancedDegradationSettings/"
3883 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 13:13:043884 SetupTest();
Åsa Persson1b247f12019-08-14 15:26:393885
Åsa Persson1b247f12019-08-14 15:26:393886 const int kMinBitrateBps = 425000;
3887 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 13:13:043888 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 15:26:393889
Åsa Persson45b176f2019-09-30 09:19:053890 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:393891 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 15:26:393892 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3893
3894 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3895 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053896 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:393897 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 15:26:393898 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3899
3900 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3901 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053902 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:543903 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:393904 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3905
Åsa Persson30ab0152019-08-27 10:22:333906 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3907 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053908 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:543909 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 13:13:043910 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 10:22:333911 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3912
3913 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 15:26:393914 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053915 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:333916 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 15:26:393917
Åsa Persson30ab0152019-08-27 10:22:333918 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043919 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 15:26:393920 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053921 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043922 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 10:22:333923 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3924
3925 video_stream_encoder_->Stop();
3926}
3927
Åsa Perssonccfb3402019-09-25 13:13:043928TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 09:19:053929 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3930 test::ScopedFieldTrials field_trials(
3931 "WebRTC-Video-BalancedDegradationSettings/"
3932 "pixels:57600|129600|230400,fps:7|24|24/");
3933 SetupTest();
3934 OnBitrateUpdated(kLowTargetBitrateBps);
3935
Evan Shrubsole5cd7eb82020-05-25 12:08:393936 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 09:19:053937
3938 // Insert frame, expect scaled down:
3939 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3940 InsertFrame();
3941 EXPECT_FALSE(WaitForFrame(1000));
3942 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3943 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3944
3945 // Insert frame, expect scaled down:
3946 // resolution (320x180@24fps).
3947 InsertFrame();
3948 EXPECT_FALSE(WaitForFrame(1000));
3949 EXPECT_LT(source_.sink_wants().max_pixel_count,
3950 source_.last_wants().max_pixel_count);
3951 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3952
3953 // Frame should not be dropped (min pixels per frame reached).
3954 InsertFrameAndWaitForEncoded();
3955
3956 video_stream_encoder_->Stop();
3957}
3958
3959TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:333960 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:323961 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:333962 "WebRTC-Video-BalancedDegradationSettings/"
3963 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:043964 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:333965
Åsa Persson30ab0152019-08-27 10:22:333966 const int kResolutionMinBitrateBps = 435000;
3967 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 13:13:043968 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333969
Åsa Persson45b176f2019-09-30 09:19:053970 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:393971 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:333972 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3973
3974 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3975 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053976 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:393977 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:333978 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3979
3980 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3981 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053982 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:543983 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:333984 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3985
3986 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3987 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053988 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:543989 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 15:26:393990 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3991
Åsa Persson30ab0152019-08-27 10:22:333992 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3993 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053994 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:543995 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:333996 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3997
3998 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3999 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054000 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334001 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4002
4003 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044004 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334005 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054006 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544007 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334008 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4009
4010 video_stream_encoder_->Stop();
4011}
4012
Åsa Perssonccfb3402019-09-25 13:13:044013TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:334014 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:324015 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:334016 "WebRTC-Video-BalancedDegradationSettings/"
4017 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:044018 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:334019
Åsa Persson30ab0152019-08-27 10:22:334020 const int kMinBitrateBps = 425000;
4021 const int kTooLowMinBitrateBps = 424000;
4022 const int kResolutionMinBitrateBps = 435000;
4023 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 13:13:044024 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334025
Åsa Persson45b176f2019-09-30 09:19:054026 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394027 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 10:22:334028 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4029
4030 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4031 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054032 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 12:08:394033 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 10:22:334034 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4035
4036 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4037 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054038 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544039 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334040 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4041
4042 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4043 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:054044 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544045 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334046 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4047
4048 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4049 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054050 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334051 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4052
4053 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044054 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334055 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054056 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544057 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334058 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4059
4060 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044061 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334062 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054063 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:334064 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4065
4066 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:044067 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:334068 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:054069 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 14:19:544070 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 10:22:334071 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4072
Åsa Persson1b247f12019-08-14 15:26:394073 video_stream_encoder_->Stop();
4074}
4075
mflodmancc3d4422017-08-03 15:27:514076TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:314077 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4078 const int kWidth = 1280;
4079 const int kHeight = 720;
Henrik Boström381d1092020-05-12 16:49:074080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074081 DataRate::BitsPerSec(kTargetBitrateBps),
4082 DataRate::BitsPerSec(kTargetBitrateBps),
4083 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:314084
Taylor Brandstetter49fcc102018-05-16 21:20:414085 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:234086 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 08:47:314087 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:514088 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414089 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:314090
Åsa Persson8c1bf952018-09-13 08:42:194091 int64_t timestamp_ms = kFrameIntervalMs;
4092 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524093 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:394094 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314095 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4096 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4097 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4098 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4099
4100 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 15:27:514101 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194102 timestamp_ms += kFrameIntervalMs;
4103 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4104 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:394105 EXPECT_THAT(source.sink_wants(),
4106 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 08:47:314107 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4108 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4109 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4110 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4111
4112 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 15:27:514113 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194114 timestamp_ms += kFrameIntervalMs;
4115 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4116 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544117 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314118 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4119 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4120 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4121 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4122
Jonathan Yubc771b72017-12-09 01:04:294123 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 15:27:514124 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194125 timestamp_ms += kFrameIntervalMs;
4126 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4127 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544128 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314129 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4130 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294131 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:314132 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4133
Jonathan Yubc771b72017-12-09 01:04:294134 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 15:27:514135 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194136 timestamp_ms += kFrameIntervalMs;
4137 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4138 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544139 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294140 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 08:47:314141 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4142 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4143 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4144 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4145
Jonathan Yubc771b72017-12-09 01:04:294146 // Trigger quality adapt down, expect no change (min resolution reached).
4147 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:194148 timestamp_ms += kFrameIntervalMs;
4149 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4150 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544151 EXPECT_THAT(source.sink_wants(), FpsMax());
4152 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-09 01:04:294153 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4154 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4155 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4156 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4157
Evan Shrubsole64469032020-06-11 08:45:294158 // Trigger quality adapt up, expect upscaled resolution (480x270).
4159 video_stream_encoder_->TriggerQualityHigh();
4160 timestamp_ms += kFrameIntervalMs;
4161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4162 WaitForEncodedFrame(timestamp_ms);
4163 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4164 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4165 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4166 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4167 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4168
4169 // Trigger quality and cpu adapt up since both are most limited, expect
4170 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 10:24:334171 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294172 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194173 timestamp_ms += kFrameIntervalMs;
4174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4175 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544176 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-09 01:04:294177 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4178 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4179 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294180 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:294181
Evan Shrubsole64469032020-06-11 08:45:294182 // Trigger quality and cpu adapt up since both are most limited, expect
4183 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 10:24:334184 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 08:45:294185 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194186 timestamp_ms += kFrameIntervalMs;
4187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4188 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544189 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 08:47:314190 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 08:45:294191 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 08:47:314192 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:294193 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4194 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314195
Evan Shrubsole64469032020-06-11 08:45:294196 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4197 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 10:24:334198 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 08:42:194199 timestamp_ms += kFrameIntervalMs;
4200 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4201 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:544202 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 08:47:314203 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4204 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294205 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294206 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 08:47:314207
4208 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 15:27:514209 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194210 timestamp_ms += kFrameIntervalMs;
4211 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524212 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:544213 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:394214 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:314215 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4216 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:294217 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:294218 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 11:59:514219
mflodmancc3d4422017-08-03 15:27:514220 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 11:59:514221}
4222
mflodmancc3d4422017-08-03 15:27:514223TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 12:51:494224 const int kWidth = 640;
4225 const int kHeight = 360;
perkj803d97f2016-11-01 18:45:464226
Henrik Boström381d1092020-05-12 16:49:074227 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074228 DataRate::BitsPerSec(kTargetBitrateBps),
4229 DataRate::BitsPerSec(kTargetBitrateBps),
4230 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:524231
perkj803d97f2016-11-01 18:45:464232 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494233 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524234 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 18:45:464235 }
4236
mflodmancc3d4422017-08-03 15:27:514237 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:464238 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:494239 video_source_.IncomingCapturedFrame(CreateFrame(
4240 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524241 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 18:45:464242 }
4243
mflodmancc3d4422017-08-03 15:27:514244 video_stream_encoder_->Stop();
4245 video_stream_encoder_.reset();
perkj803d97f2016-11-01 18:45:464246 stats_proxy_.reset();
sprangf8ee65e2017-02-28 16:49:334247
Ying Wangef3998f2019-12-09 12:06:534248 EXPECT_METRIC_EQ(
4249 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4250 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 18:45:464251 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4252}
4253
mflodmancc3d4422017-08-03 15:27:514254TEST_F(VideoStreamEncoderTest,
4255 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 16:49:074256 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074257 DataRate::BitsPerSec(kTargetBitrateBps),
4258 DataRate::BitsPerSec(kTargetBitrateBps),
4259 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 09:01:064260 const int kWidth = 640;
4261 const int kHeight = 360;
4262
Taylor Brandstetter49fcc102018-05-16 21:20:414263 video_stream_encoder_->SetSource(&video_source_,
4264 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 09:01:064265
4266 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4267 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524268 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 09:01:064269 }
4270
mflodmancc3d4422017-08-03 15:27:514271 video_stream_encoder_->Stop();
4272 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 09:01:064273 stats_proxy_.reset();
4274
4275 EXPECT_EQ(0,
4276 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4277}
4278
Per Kjellanderdcef6412020-10-07 13:09:054279TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4280 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034281 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:054282 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 14:24:024283
4284 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 10:32:224285 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 14:36:494286 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 13:16:284287 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4288 kDefaultFps));
sprang57c2fff2017-01-16 14:24:024289
Henrik Boström381d1092020-05-12 16:49:074290 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074291 DataRate::BitsPerSec(kLowTargetBitrateBps),
4292 DataRate::BitsPerSec(kLowTargetBitrateBps),
4293 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 14:24:024294
sprang57c2fff2017-01-16 14:24:024295 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234296 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4297 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054298 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4299 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4300
Erik Språngd7329ca2019-02-21 20:19:534301 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 15:44:424302 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 12:31:234303 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024304
Per Kjellanderdcef6412020-10-07 13:09:054305 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 14:24:024306 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234307 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4308 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 13:09:054309 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 12:31:234310 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:024311
Per Kjellanderdcef6412020-10-07 13:09:054312 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 12:31:234313 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 09:28:414314 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 20:19:534315 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 12:31:234316 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4317 WaitForEncodedFrame(CurrentTimeMs());
4318 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 20:19:534319 }
Per Kjellanderdcef6412020-10-07 13:09:054320 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 20:19:534321
mflodmancc3d4422017-08-03 15:27:514322 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 14:24:024323}
4324
Per Kjellanderf86cf4c2020-12-30 14:27:354325TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 15:53:224326 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034327 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:224328 kVideoLayersAllocation);
4329
4330 const int kDefaultFps = 30;
4331
4332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4333 DataRate::BitsPerSec(kLowTargetBitrateBps),
4334 DataRate::BitsPerSec(kLowTargetBitrateBps),
4335 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4336
4337 video_source_.IncomingCapturedFrame(
4338 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4339 WaitForEncodedFrame(CurrentTimeMs());
4340 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4341 VideoLayersAllocation last_layer_allocation =
4342 sink_.GetLastVideoLayersAllocation();
4343 // kLowTargetBitrateBps is only enough for one spatial layer.
4344 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4345
4346 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 15:44:424347 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 15:53:224348 // Check that encoder has been updated too, not just allocation observer.
4349 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4350 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4351
Erik Språng9d69cbe2020-10-22 15:44:424352 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 15:53:224353 int number_of_layers_allocation = 1;
4354 const int64_t start_time_ms = CurrentTimeMs();
4355 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4356 video_source_.IncomingCapturedFrame(
4357 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4358 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 15:53:224359 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4360 number_of_layers_allocation = sink_.number_of_layers_allocations();
4361 VideoLayersAllocation new_allocation =
4362 sink_.GetLastVideoLayersAllocation();
4363 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4364 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4365 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4366 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4367 .target_bitrate_per_temporal_layer,
4368 last_layer_allocation.active_spatial_layers[0]
4369 .target_bitrate_per_temporal_layer);
4370 last_layer_allocation = new_allocation;
4371 }
4372 }
4373 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4374 video_stream_encoder_->Stop();
4375}
4376
4377TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134378 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354379 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4380 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4381 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354382 VideoEncoderConfig video_encoder_config;
4383 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4384 /* num_streams*/ 3, &video_encoder_config);
4385 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4386 video_encoder_config.content_type =
4387 VideoEncoderConfig::ContentType::kRealtimeVideo;
4388 video_encoder_config.encoder_specific_settings =
4389 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4390 VideoEncoder::GetDefaultVp8Settings());
4391 for (auto& layer : video_encoder_config.simulcast_layers) {
4392 layer.num_temporal_layers = 2;
4393 }
4394 // Simulcast layers are used for enabling/disabling streams.
4395 video_encoder_config.simulcast_layers[0].active = true;
4396 video_encoder_config.simulcast_layers[1].active = false;
4397 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034398 ConfigureEncoder(std::move(video_encoder_config),
4399 VideoStreamEncoder::BitrateAllocationCallbackType::
4400 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354401
4402 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4403 DataRate::BitsPerSec(kTargetBitrateBps),
4404 DataRate::BitsPerSec(kTargetBitrateBps),
4405 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4406
4407 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4408 WaitForEncodedFrame(CurrentTimeMs());
4409 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4410 VideoLayersAllocation last_layer_allocation =
4411 sink_.GetLastVideoLayersAllocation();
4412
4413 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4414 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4415 .target_bitrate_per_temporal_layer,
4416 SizeIs(2));
4417 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4418 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4419 video_stream_encoder_->Stop();
4420}
4421
4422TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 14:36:134423 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 14:27:354424 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4425 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4426 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354427 VideoEncoderConfig video_encoder_config;
4428 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4429 /* num_streams*/ 3, &video_encoder_config);
4430 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4431 video_encoder_config.content_type =
4432 VideoEncoderConfig::ContentType::kRealtimeVideo;
4433 video_encoder_config.encoder_specific_settings =
4434 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4435 VideoEncoder::GetDefaultVp8Settings());
4436 for (auto& layer : video_encoder_config.simulcast_layers) {
4437 layer.num_temporal_layers = 2;
4438 }
4439 // Simulcast layers are used for enabling/disabling streams.
4440 video_encoder_config.simulcast_layers[0].active = true;
4441 video_encoder_config.simulcast_layers[1].active = false;
4442 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034443 ConfigureEncoder(std::move(video_encoder_config),
4444 VideoStreamEncoder::BitrateAllocationCallbackType::
4445 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354446
4447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4448 DataRate::BitsPerSec(kTargetBitrateBps),
4449 DataRate::BitsPerSec(kTargetBitrateBps),
4450 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4451
4452 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4453 WaitForEncodedFrame(CurrentTimeMs());
4454 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4455 VideoLayersAllocation last_layer_allocation =
4456 sink_.GetLastVideoLayersAllocation();
4457
4458 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4459 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4460 .target_bitrate_per_temporal_layer,
4461 SizeIs(2));
4462 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4463
4464 video_stream_encoder_->Stop();
4465}
4466
4467TEST_F(VideoStreamEncoderTest,
4468 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4469 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4470 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354471 VideoEncoderConfig video_encoder_config;
4472 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4473 /* num_streams*/ 1, &video_encoder_config);
4474 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4475 video_encoder_config.content_type =
4476 VideoEncoderConfig::ContentType::kRealtimeVideo;
4477 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4478 vp9_settings.numberOfSpatialLayers = 2;
4479 vp9_settings.numberOfTemporalLayers = 2;
4480 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4481 vp9_settings.automaticResizeOn = false;
4482 video_encoder_config.encoder_specific_settings =
4483 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4484 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034485 ConfigureEncoder(std::move(video_encoder_config),
4486 VideoStreamEncoder::BitrateAllocationCallbackType::
4487 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354488
4489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4490 DataRate::BitsPerSec(kTargetBitrateBps),
4491 DataRate::BitsPerSec(kTargetBitrateBps),
4492 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4493
4494 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4495 WaitForEncodedFrame(CurrentTimeMs());
4496 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4497 VideoLayersAllocation last_layer_allocation =
4498 sink_.GetLastVideoLayersAllocation();
4499
4500 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4501 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4502 .target_bitrate_per_temporal_layer,
4503 SizeIs(2));
4504 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4505 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4506 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4507 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4508 .target_bitrate_per_temporal_layer,
4509 SizeIs(2));
4510 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4511 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4512 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4513
4514 // Since full SVC is used, expect the top layer to utilize the full target
4515 // rate.
4516 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4517 .target_bitrate_per_temporal_layer[1],
4518 DataRate::BitsPerSec(kTargetBitrateBps));
4519 video_stream_encoder_->Stop();
4520}
4521
4522TEST_F(VideoStreamEncoderTest,
4523 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4524 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4525 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 14:27:354526 VideoEncoderConfig video_encoder_config;
4527 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4528 /* num_streams*/ 1, &video_encoder_config);
4529 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4530 video_encoder_config.content_type =
4531 VideoEncoderConfig::ContentType::kRealtimeVideo;
4532 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4533 vp9_settings.numberOfSpatialLayers = 2;
4534 vp9_settings.numberOfTemporalLayers = 2;
4535 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4536 vp9_settings.automaticResizeOn = false;
4537 video_encoder_config.encoder_specific_settings =
4538 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4539 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034540 ConfigureEncoder(std::move(video_encoder_config),
4541 VideoStreamEncoder::BitrateAllocationCallbackType::
4542 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354543
4544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4545 DataRate::BitsPerSec(kTargetBitrateBps),
4546 DataRate::BitsPerSec(kTargetBitrateBps),
4547 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4548
4549 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4550 WaitForEncodedFrame(CurrentTimeMs());
4551 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4552 VideoLayersAllocation last_layer_allocation =
4553 sink_.GetLastVideoLayersAllocation();
4554
4555 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4556 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4557 .target_bitrate_per_temporal_layer,
4558 SizeIs(1));
4559 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4560 .target_bitrate_per_temporal_layer,
4561 SizeIs(1));
4562 // Since full SVC is used, expect the top layer to utilize the full target
4563 // rate.
4564 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4565 .target_bitrate_per_temporal_layer[0],
4566 DataRate::BitsPerSec(kTargetBitrateBps));
4567 video_stream_encoder_->Stop();
4568}
4569
4570TEST_F(VideoStreamEncoderTest,
4571 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4572 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4573 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354574 VideoEncoderConfig video_encoder_config;
4575 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4576 /* num_streams*/ 1, &video_encoder_config);
4577 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4578 video_encoder_config.content_type =
4579 VideoEncoderConfig::ContentType::kRealtimeVideo;
4580 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4581 vp9_settings.numberOfSpatialLayers = 2;
4582 vp9_settings.numberOfTemporalLayers = 2;
4583 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4584 vp9_settings.automaticResizeOn = false;
4585 video_encoder_config.encoder_specific_settings =
4586 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4587 vp9_settings);
Per Kjellanderb03b6c82021-01-03 09:26:034588 ConfigureEncoder(std::move(video_encoder_config),
4589 VideoStreamEncoder::BitrateAllocationCallbackType::
4590 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354591
4592 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4593 DataRate::BitsPerSec(kTargetBitrateBps),
4594 DataRate::BitsPerSec(kTargetBitrateBps),
4595 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4596
4597 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4598 WaitForEncodedFrame(CurrentTimeMs());
4599 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4600 VideoLayersAllocation last_layer_allocation =
4601 sink_.GetLastVideoLayersAllocation();
4602
4603 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4604 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4605 .target_bitrate_per_temporal_layer,
4606 SizeIs(2));
4607 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4608 .target_bitrate_per_temporal_layer,
4609 SizeIs(2));
4610 // Since KSVC is, spatial layers are independend except on key frames.
4611 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4612 .target_bitrate_per_temporal_layer[1],
4613 DataRate::BitsPerSec(kTargetBitrateBps));
4614 video_stream_encoder_->Stop();
4615}
4616
4617TEST_F(VideoStreamEncoderTest,
4618 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4619 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4620 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4621 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354622 VideoEncoderConfig video_encoder_config;
4623 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4624 /* num_streams*/ 1, &video_encoder_config);
4625 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4626 video_encoder_config.content_type =
4627 VideoEncoderConfig::ContentType::kRealtimeVideo;
4628 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4629 vp9_settings.numberOfSpatialLayers = 3;
4630 vp9_settings.numberOfTemporalLayers = 2;
4631 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4632 vp9_settings.automaticResizeOn = false;
4633 video_encoder_config.encoder_specific_settings =
4634 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4635 vp9_settings);
4636 // Simulcast layers are used for enabling/disabling streams.
4637 video_encoder_config.simulcast_layers.resize(3);
4638 video_encoder_config.simulcast_layers[0].active = false;
4639 video_encoder_config.simulcast_layers[1].active = true;
4640 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034641 ConfigureEncoder(std::move(video_encoder_config),
4642 VideoStreamEncoder::BitrateAllocationCallbackType::
4643 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354644
4645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4646 DataRate::BitsPerSec(kTargetBitrateBps),
4647 DataRate::BitsPerSec(kTargetBitrateBps),
4648 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4649
4650 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4651 WaitForEncodedFrame(CurrentTimeMs());
4652 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4653 VideoLayersAllocation last_layer_allocation =
4654 sink_.GetLastVideoLayersAllocation();
4655
4656 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4657 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4658 .target_bitrate_per_temporal_layer,
4659 SizeIs(2));
4660 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4661 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4662
4663 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4664 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4665 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4666 .target_bitrate_per_temporal_layer,
4667 SizeIs(2));
4668 // Since full SVC is used, expect the top layer to utilize the full target
4669 // rate.
4670 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4671 .target_bitrate_per_temporal_layer[1],
4672 DataRate::BitsPerSec(kTargetBitrateBps));
4673 video_stream_encoder_->Stop();
4674}
4675
4676TEST_F(VideoStreamEncoderTest,
4677 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4678 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4679 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4680 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354681 VideoEncoderConfig video_encoder_config;
4682 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4683 /* num_streams*/ 1, &video_encoder_config);
4684 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4685 video_encoder_config.content_type =
4686 VideoEncoderConfig::ContentType::kRealtimeVideo;
4687 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4688 vp9_settings.numberOfSpatialLayers = 3;
4689 vp9_settings.numberOfTemporalLayers = 2;
4690 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4691 vp9_settings.automaticResizeOn = false;
4692 video_encoder_config.encoder_specific_settings =
4693 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4694 vp9_settings);
4695 // Simulcast layers are used for enabling/disabling streams.
4696 video_encoder_config.simulcast_layers.resize(3);
4697 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 09:26:034698 ConfigureEncoder(std::move(video_encoder_config),
4699 VideoStreamEncoder::BitrateAllocationCallbackType::
4700 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354701
4702 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4703 DataRate::BitsPerSec(kTargetBitrateBps),
4704 DataRate::BitsPerSec(kTargetBitrateBps),
4705 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4706
4707 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4708 WaitForEncodedFrame(CurrentTimeMs());
4709 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4710 VideoLayersAllocation last_layer_allocation =
4711 sink_.GetLastVideoLayersAllocation();
4712
4713 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4714 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4715 .target_bitrate_per_temporal_layer,
4716 SizeIs(2));
4717 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4718 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4719
4720 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4721 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4722 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4723 .target_bitrate_per_temporal_layer,
4724 SizeIs(2));
4725 video_stream_encoder_->Stop();
4726}
4727
4728TEST_F(VideoStreamEncoderTest,
4729 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4730 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4731 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4732 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 14:27:354733 VideoEncoderConfig video_encoder_config;
4734 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4735 /* num_streams*/ 1, &video_encoder_config);
4736 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4737 video_encoder_config.content_type =
4738 VideoEncoderConfig::ContentType::kRealtimeVideo;
4739 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4740 vp9_settings.numberOfSpatialLayers = 3;
4741 vp9_settings.numberOfTemporalLayers = 2;
4742 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4743 vp9_settings.automaticResizeOn = false;
4744 video_encoder_config.encoder_specific_settings =
4745 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4746 vp9_settings);
4747 // Simulcast layers are used for enabling/disabling streams.
4748 video_encoder_config.simulcast_layers.resize(3);
4749 video_encoder_config.simulcast_layers[0].active = false;
4750 video_encoder_config.simulcast_layers[1].active = false;
4751 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 09:26:034752 ConfigureEncoder(std::move(video_encoder_config),
4753 VideoStreamEncoder::BitrateAllocationCallbackType::
4754 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 14:27:354755
4756 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4757 DataRate::BitsPerSec(kTargetBitrateBps),
4758 DataRate::BitsPerSec(kTargetBitrateBps),
4759 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4760
4761 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4762 WaitForEncodedFrame(CurrentTimeMs());
4763 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4764 VideoLayersAllocation last_layer_allocation =
4765 sink_.GetLastVideoLayersAllocation();
4766
4767 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4768 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4769 .target_bitrate_per_temporal_layer,
4770 SizeIs(2));
4771 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4772 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4773 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4774 .target_bitrate_per_temporal_layer[1],
4775 DataRate::BitsPerSec(kTargetBitrateBps));
4776 video_stream_encoder_->Stop();
4777}
4778
4779TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4780 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:034781 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 14:27:354782 kVideoLayersAllocation);
4783 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4784 DataRate::BitsPerSec(kTargetBitrateBps),
4785 DataRate::BitsPerSec(kTargetBitrateBps),
4786 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4787
4788 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4789 WaitForEncodedFrame(CurrentTimeMs());
4790 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4791 VideoLayersAllocation last_layer_allocation =
4792 sink_.GetLastVideoLayersAllocation();
4793
4794 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4795 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4796 .target_bitrate_per_temporal_layer,
4797 SizeIs(1));
4798 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4799 .target_bitrate_per_temporal_layer[0],
4800 DataRate::BitsPerSec(kTargetBitrateBps));
4801 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4802 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4803 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4804 video_stream_encoder_->Stop();
4805}
4806
4807TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 15:53:224808 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4809 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034810 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 15:53:224811 kVideoLayersAllocation);
4812
4813 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4814 DataRate::BitsPerSec(kLowTargetBitrateBps),
4815 DataRate::BitsPerSec(kLowTargetBitrateBps),
4816 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4817
4818 video_source_.IncomingCapturedFrame(
4819 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4820 WaitForEncodedFrame(CurrentTimeMs());
4821 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4822 VideoLayersAllocation last_layer_allocation =
4823 sink_.GetLastVideoLayersAllocation();
4824 // kLowTargetBitrateBps is only enough for one spatial layer.
4825 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4826 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4827 .target_bitrate_per_temporal_layer[0],
4828 DataRate::BitsPerSec(kLowTargetBitrateBps));
4829
4830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4831 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4832 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4833 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4834 video_source_.IncomingCapturedFrame(
4835 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4836 WaitForEncodedFrame(CurrentTimeMs());
4837
4838 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4839 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4840 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4841 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4842 .target_bitrate_per_temporal_layer[0],
4843 DataRate::Zero());
4844
4845 video_stream_encoder_->Stop();
4846}
4847
Per Kjellander4190ce92020-12-15 16:24:554848TEST_F(VideoStreamEncoderTest,
4849 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4850 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034851 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 16:24:554852 kVideoLayersAllocation);
4853
4854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4855 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4856 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4857 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4858
4859 video_source_.IncomingCapturedFrame(
4860 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4861 WaitForEncodedFrame(CurrentTimeMs());
4862 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4863 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4864 SizeIs(2));
4865 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4866 codec_width_);
4867 EXPECT_EQ(
4868 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4869 codec_height_);
4870
4871 video_source_.IncomingCapturedFrame(
4872 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4873 WaitForEncodedFrame(CurrentTimeMs());
4874 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4875 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4876 SizeIs(2));
4877 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4878 codec_width_ / 2);
4879 EXPECT_EQ(
4880 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4881 codec_height_ / 2);
4882
4883 video_stream_encoder_->Stop();
4884}
4885
Åsa Perssonc29cb2c2019-03-25 11:06:594886TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4887 // 2 TLs configured, temporal layers supported by encoder.
4888 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 13:09:054889 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034890 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:054891 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:594892 fake_encoder_.SetTemporalLayersSupported(0, true);
4893
4894 // Bitrate allocated across temporal layers.
4895 const int kTl0Bps = kTargetBitrateBps *
4896 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:464897 kNumTemporalLayers, /*temporal_id*/ 0,
4898 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:594899 const int kTl1Bps = kTargetBitrateBps *
4900 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:464901 kNumTemporalLayers, /*temporal_id*/ 1,
4902 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:594903 VideoBitrateAllocation expected_bitrate;
4904 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4905 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4906
4907 VerifyAllocatedBitrate(expected_bitrate);
4908 video_stream_encoder_->Stop();
4909}
4910
4911TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4912 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 13:09:054913 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034914 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:054915 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:594916 fake_encoder_.SetTemporalLayersSupported(0, false);
4917
4918 // Temporal layers not supported by the encoder.
4919 // Total bitrate should be at ti:0.
4920 VideoBitrateAllocation expected_bitrate;
4921 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4922
4923 VerifyAllocatedBitrate(expected_bitrate);
4924 video_stream_encoder_->Stop();
4925}
4926
4927TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 13:09:054928 webrtc::test::ScopedFieldTrials field_trials(
4929 "WebRTC-Video-QualityScalerSettings/"
4930 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4931 // Reset encoder for field trials to take effect.
4932 ConfigureEncoder(video_encoder_config_.Copy());
4933
Åsa Perssonc29cb2c2019-03-25 11:06:594934 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 13:09:054935 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 09:26:034936 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:054937 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 11:06:594938 fake_encoder_.SetTemporalLayersSupported(0, true);
4939 fake_encoder_.SetTemporalLayersSupported(1, false);
4940
4941 const int kS0Bps = 150000;
4942 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:464943 kS0Bps *
4944 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4945 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:594946 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:464947 kS0Bps *
4948 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4949 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:594950 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4951 // Temporal layers not supported by si:1.
4952 VideoBitrateAllocation expected_bitrate;
4953 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4954 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4955 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4956
4957 VerifyAllocatedBitrate(expected_bitrate);
4958 video_stream_encoder_->Stop();
4959}
4960
Niels Möller7dc26b72017-12-06 09:27:484961TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4962 const int kFrameWidth = 1280;
4963 const int kFrameHeight = 720;
4964 const int kFramerate = 24;
4965
Henrik Boström381d1092020-05-12 16:49:074966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:074967 DataRate::BitsPerSec(kTargetBitrateBps),
4968 DataRate::BitsPerSec(kTargetBitrateBps),
4969 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:484970 test::FrameForwarder source;
4971 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414972 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:484973
4974 // Insert a single frame, triggering initial configuration.
4975 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4976 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4977
4978 EXPECT_EQ(
4979 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4980 kDefaultFramerate);
4981
4982 // Trigger reconfigure encoder (without resetting the entire instance).
4983 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:514984 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4985 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 09:27:484986 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 09:27:484987 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:474988 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:484989 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4990
4991 // Detector should be updated with fps limit from codec config.
4992 EXPECT_EQ(
4993 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4994 kFramerate);
4995
4996 // Trigger overuse, max framerate should be reduced.
4997 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4998 stats.input_frame_rate = kFramerate;
4999 stats_proxy_->SetMockStats(stats);
5000 video_stream_encoder_->TriggerCpuOveruse();
5001 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5002 int adapted_framerate =
5003 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5004 EXPECT_LT(adapted_framerate, kFramerate);
5005
5006 // Trigger underuse, max framerate should go back to codec configured fps.
5007 // Set extra low fps, to make sure it's actually reset, not just incremented.
5008 stats = stats_proxy_->GetStats();
5009 stats.input_frame_rate = adapted_framerate / 2;
5010 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335011 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485012 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5013 EXPECT_EQ(
5014 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5015 kFramerate);
5016
5017 video_stream_encoder_->Stop();
5018}
5019
5020TEST_F(VideoStreamEncoderTest,
5021 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5022 const int kFrameWidth = 1280;
5023 const int kFrameHeight = 720;
5024 const int kLowFramerate = 15;
5025 const int kHighFramerate = 25;
5026
Henrik Boström381d1092020-05-12 16:49:075027 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075028 DataRate::BitsPerSec(kTargetBitrateBps),
5029 DataRate::BitsPerSec(kTargetBitrateBps),
5030 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:485031 test::FrameForwarder source;
5032 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415033 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:485034
5035 // Trigger initial configuration.
5036 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515037 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5038 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 09:27:485039 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 09:27:485040 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 06:57:515041 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 07:51:475042 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485043 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5044
5045 EXPECT_EQ(
5046 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5047 kLowFramerate);
5048
5049 // Trigger overuse, max framerate should be reduced.
5050 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5051 stats.input_frame_rate = kLowFramerate;
5052 stats_proxy_->SetMockStats(stats);
5053 video_stream_encoder_->TriggerCpuOveruse();
5054 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5055 int adapted_framerate =
5056 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5057 EXPECT_LT(adapted_framerate, kLowFramerate);
5058
5059 // Reconfigure the encoder with a new (higher max framerate), max fps should
5060 // still respect the adaptation.
Åsa Persson17107062020-10-08 06:57:515061 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 09:27:485062 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5063 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475064 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:485065 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5066
5067 EXPECT_EQ(
5068 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5069 adapted_framerate);
5070
5071 // Trigger underuse, max framerate should go back to codec configured fps.
5072 stats = stats_proxy_->GetStats();
5073 stats.input_frame_rate = adapted_framerate;
5074 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 10:24:335075 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 09:27:485076 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5077 EXPECT_EQ(
5078 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5079 kHighFramerate);
5080
5081 video_stream_encoder_->Stop();
5082}
5083
mflodmancc3d4422017-08-03 15:27:515084TEST_F(VideoStreamEncoderTest,
5085 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 11:21:075086 const int kFrameWidth = 1280;
5087 const int kFrameHeight = 720;
5088 const int kFramerate = 24;
5089
Henrik Boström381d1092020-05-12 16:49:075090 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075091 DataRate::BitsPerSec(kTargetBitrateBps),
5092 DataRate::BitsPerSec(kTargetBitrateBps),
5093 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 11:21:075094 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:515095 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415096 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 11:21:075097
5098 // Trigger initial configuration.
5099 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 06:57:515100 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5101 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 11:21:075102 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 11:21:075103 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 15:27:515104 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475105 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:515106 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 11:21:075107
Niels Möller7dc26b72017-12-06 09:27:485108 EXPECT_EQ(
5109 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5110 kFramerate);
sprangfda496a2017-06-15 11:21:075111
5112 // Trigger overuse, max framerate should be reduced.
5113 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5114 stats.input_frame_rate = kFramerate;
5115 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 15:27:515116 video_stream_encoder_->TriggerCpuOveruse();
5117 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:485118 int adapted_framerate =
5119 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 11:21:075120 EXPECT_LT(adapted_framerate, kFramerate);
5121
5122 // Change degradation preference to not enable framerate scaling. Target
5123 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 16:49:075124 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 21:20:415125 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 09:27:485126 EXPECT_EQ(
5127 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5128 kFramerate);
sprangfda496a2017-06-15 11:21:075129
mflodmancc3d4422017-08-03 15:27:515130 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 11:21:075131}
5132
mflodmancc3d4422017-08-03 15:27:515133TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495134 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075135 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075136 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5137 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5138 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495139 const int kWidth = 640;
5140 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225141
asaperssonfab67072017-04-04 12:51:495142 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225143
5144 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525145 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225146
5147 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095148 EXPECT_TRUE_WAIT(
5149 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 15:02:225150
sprangc5d62e22017-04-03 06:53:045151 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 15:02:225152
asaperssonfab67072017-04-04 12:51:495153 // Next frame is scaled.
kthelgason2bc68642017-02-07 15:02:225154 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:495155 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 15:02:225156
5157 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:525158 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225159
Henrik Boström2671dac2020-05-19 14:29:095160 EXPECT_TRUE_WAIT(
5161 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 15:02:225162
mflodmancc3d4422017-08-03 15:27:515163 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225164}
5165
mflodmancc3d4422017-08-03 15:27:515166TEST_F(VideoStreamEncoderTest,
5167 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:495168 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 16:49:075169 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075170 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5171 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5172 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:495173 const int kWidth = 640;
5174 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:225175
5176 // We expect the n initial frames to get dropped.
5177 int i;
5178 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 12:51:495179 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525180 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:225181 }
5182 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 12:51:495183 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:525184 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 15:02:225185
5186 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 12:51:495187 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:225188
mflodmancc3d4422017-08-03 15:27:515189 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225190}
5191
mflodmancc3d4422017-08-03 15:27:515192TEST_F(VideoStreamEncoderTest,
5193 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 12:51:495194 const int kWidth = 640;
5195 const int kHeight = 360;
Henrik Boström381d1092020-05-12 16:49:075196 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075197 DataRate::BitsPerSec(kLowTargetBitrateBps),
5198 DataRate::BitsPerSec(kLowTargetBitrateBps),
5199 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 15:02:225200
5201 // Set degradation preference.
mflodmancc3d4422017-08-03 15:27:515202 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415203 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 15:02:225204
asaperssonfab67072017-04-04 12:51:495205 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:225206 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525207 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 15:02:225208
mflodmancc3d4422017-08-03 15:27:515209 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:225210}
5211
mflodmancc3d4422017-08-03 15:27:515212TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 12:51:495213 const int kWidth = 640;
5214 const int kHeight = 360;
kthelgasonad9010c2017-02-14 08:46:515215 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:135216
5217 VideoEncoderConfig video_encoder_config;
5218 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5219 // Make format different, to force recreation of encoder.
5220 video_encoder_config.video_format.parameters["foo"] = "foo";
5221 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:475222 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 16:49:075223 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075224 DataRate::BitsPerSec(kLowTargetBitrateBps),
5225 DataRate::BitsPerSec(kLowTargetBitrateBps),
5226 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:185227
kthelgasonb83797b2017-02-14 19:57:255228 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 21:20:415229 video_stream_encoder_->SetSource(&video_source_,
5230 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 08:46:515231
asaperssonfab67072017-04-04 12:51:495232 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 08:46:515233 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:525234 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 08:46:515235
mflodmancc3d4422017-08-03 15:27:515236 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 08:46:515237 fake_encoder_.SetQualityScaling(true);
5238}
5239
Åsa Persson139f4dc2019-08-02 07:29:585240TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5241 webrtc::test::ScopedFieldTrials field_trials(
5242 "WebRTC-Video-QualityScalerSettings/"
5243 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5244 // Reset encoder for field trials to take effect.
5245 ConfigureEncoder(video_encoder_config_.Copy());
5246 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5247 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5248 const int kWidth = 640;
5249 const int kHeight = 360;
5250
Henrik Boström381d1092020-05-12 16:49:075251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075252 DataRate::BitsPerSec(kTargetBitrateBps),
5253 DataRate::BitsPerSec(kTargetBitrateBps),
5254 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585255 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5256 // Frame should not be dropped.
5257 WaitForEncodedFrame(1);
5258
Henrik Boström381d1092020-05-12 16:49:075259 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075260 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5261 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5262 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585263 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5264 // Frame should not be dropped.
5265 WaitForEncodedFrame(2);
5266
Henrik Boström381d1092020-05-12 16:49:075267 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075268 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5269 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5270 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:585271 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5272 // Expect to drop this frame, the wait should time out.
5273 ExpectDroppedFrame();
5274
5275 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 14:29:095276 EXPECT_TRUE_WAIT(
5277 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 07:29:585278 video_stream_encoder_->Stop();
5279}
5280
Evan Shrubsolee3da1d32020-08-14 13:58:335281TEST_F(VideoStreamEncoderTest,
5282 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5283 webrtc::test::ScopedFieldTrials field_trials(
5284 "WebRTC-Video-QualityScalerSettings/"
5285 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5286 fake_encoder_.SetQualityScaling(false);
5287 ConfigureEncoder(video_encoder_config_.Copy());
5288 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5289 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5290 const int kWidth = 640;
5291 const int kHeight = 360;
5292
5293 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5294 DataRate::BitsPerSec(kTargetBitrateBps),
5295 DataRate::BitsPerSec(kTargetBitrateBps),
5296 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5297 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5298 // Frame should not be dropped.
5299 WaitForEncodedFrame(1);
5300
5301 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5302 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5303 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5304 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5305 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5306 // Frame should not be dropped.
5307 WaitForEncodedFrame(2);
5308
5309 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5310 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5311 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5312 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5313 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5314 // Not dropped since quality scaling is disabled.
5315 WaitForEncodedFrame(3);
5316
5317 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 11:12:125318 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 13:58:335319 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5320
5321 video_stream_encoder_->Stop();
5322}
5323
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475324TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5325 const int kLowTargetBitrateBps = 400000;
5326 // Set simulcast.
5327 ResetEncoder("VP8", 3, 1, 1, false);
5328 fake_encoder_.SetQualityScaling(true);
5329 const int kWidth = 1280;
5330 const int kHeight = 720;
5331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5332 DataRate::BitsPerSec(kLowTargetBitrateBps),
5333 DataRate::BitsPerSec(kLowTargetBitrateBps),
5334 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5335 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5336 // Frame should not be dropped.
5337 WaitForEncodedFrame(1);
5338
5339 // Trigger QVGA "singlecast"
5340 // Update the config.
5341 VideoEncoderConfig video_encoder_config;
5342 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5343 &video_encoder_config);
Åsa Persson7f354f82021-02-04 14:52:155344 video_encoder_config.video_stream_factory =
5345 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
5346 "VP8", /*max qp*/ 56, /*screencast*/ false,
5347 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475348 for (auto& layer : video_encoder_config.simulcast_layers) {
5349 layer.num_temporal_layers = 1;
5350 layer.max_framerate = kDefaultFramerate;
5351 }
5352 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5353 video_encoder_config.content_type =
5354 VideoEncoderConfig::ContentType::kRealtimeVideo;
5355
5356 video_encoder_config.simulcast_layers[0].active = true;
5357 video_encoder_config.simulcast_layers[1].active = false;
5358 video_encoder_config.simulcast_layers[2].active = false;
5359
5360 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5361 kMaxPayloadLength);
5362 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5363
5364 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5365 // Frame should not be dropped.
5366 WaitForEncodedFrame(2);
5367
5368 // Trigger HD "singlecast"
5369 video_encoder_config.simulcast_layers[0].active = false;
5370 video_encoder_config.simulcast_layers[1].active = false;
5371 video_encoder_config.simulcast_layers[2].active = true;
5372
5373 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5374 kMaxPayloadLength);
5375 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5376
5377 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5378 // Frame should be dropped because of initial frame drop.
5379 ExpectDroppedFrame();
5380
5381 // Expect the sink_wants to specify a scaled frame.
5382 EXPECT_TRUE_WAIT(
5383 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5384 video_stream_encoder_->Stop();
5385}
5386
Ilya Nikolaevskiycde4a9f2020-11-27 13:06:085387TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5388 const int kLowTargetBitrateBps = 400000;
5389 // Set simulcast.
5390 ResetEncoder("VP9", 1, 1, 3, false);
5391 fake_encoder_.SetQualityScaling(true);
5392 const int kWidth = 1280;
5393 const int kHeight = 720;
5394 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5395 DataRate::BitsPerSec(kLowTargetBitrateBps),
5396 DataRate::BitsPerSec(kLowTargetBitrateBps),
5397 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5398 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5399 // Frame should not be dropped.
5400 WaitForEncodedFrame(1);
5401
5402 // Trigger QVGA "singlecast"
5403 // Update the config.
5404 VideoEncoderConfig video_encoder_config;
5405 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5406 &video_encoder_config);
5407 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5408 vp9_settings.numberOfSpatialLayers = 3;
5409 // Since only one layer is active - automatic resize should be enabled.
5410 vp9_settings.automaticResizeOn = true;
5411 video_encoder_config.encoder_specific_settings =
5412 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5413 vp9_settings);
5414 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5415 video_encoder_config.content_type =
5416 VideoEncoderConfig::ContentType::kRealtimeVideo;
5417 // Currently simulcast layers |active| flags are used to inidicate
5418 // which SVC layers are active.
5419 video_encoder_config.simulcast_layers.resize(3);
5420
5421 video_encoder_config.simulcast_layers[0].active = true;
5422 video_encoder_config.simulcast_layers[1].active = false;
5423 video_encoder_config.simulcast_layers[2].active = false;
5424
5425 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5426 kMaxPayloadLength);
5427 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5428
5429 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5430 // Frame should not be dropped.
5431 WaitForEncodedFrame(2);
5432
5433 // Trigger HD "singlecast"
5434 video_encoder_config.simulcast_layers[0].active = false;
5435 video_encoder_config.simulcast_layers[1].active = false;
5436 video_encoder_config.simulcast_layers[2].active = true;
5437
5438 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5439 kMaxPayloadLength);
5440 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5441
5442 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5443 // Frame should be dropped because of initial frame drop.
5444 ExpectDroppedFrame();
5445
5446 // Expect the sink_wants to specify a scaled frame.
5447 EXPECT_TRUE_WAIT(
5448 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5449 video_stream_encoder_->Stop();
5450}
5451
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475452TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055453 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5454 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5455 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5456 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5457 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5458 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5459 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5460 fake_encoder_.SetResolutionBitrateLimits(
5461 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5462
5463 VideoEncoderConfig video_encoder_config;
5464 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5465 &video_encoder_config);
5466 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5467 vp9_settings.numberOfSpatialLayers = 3;
5468 // Since only one layer is active - automatic resize should be enabled.
5469 vp9_settings.automaticResizeOn = true;
5470 video_encoder_config.encoder_specific_settings =
5471 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5472 vp9_settings);
5473 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5474 video_encoder_config.content_type =
5475 VideoEncoderConfig::ContentType::kRealtimeVideo;
5476 // Simulcast layers are used to indicate which spatial layers are active.
5477 video_encoder_config.simulcast_layers.resize(3);
5478 video_encoder_config.simulcast_layers[0].active = false;
5479 video_encoder_config.simulcast_layers[1].active = true;
5480 video_encoder_config.simulcast_layers[2].active = false;
5481
5482 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5483 kMaxPayloadLength);
5484 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5485
5486 // The encoder bitrate limits for 360p should be used.
5487 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5488 EXPECT_FALSE(WaitForFrame(1000));
5489 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5490 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5491 VideoCodecType::kVideoCodecVP9);
5492 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5493 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5494 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5495 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5496 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5497 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5498 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5499 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5500
5501 // The encoder bitrate limits for 270p should be used.
5502 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5503 EXPECT_FALSE(WaitForFrame(1000));
5504 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5505 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5506 VideoCodecType::kVideoCodecVP9);
5507 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5508 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5509 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5510 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5511 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5512 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5513 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5514 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5515
5516 video_stream_encoder_->Stop();
5517}
5518
5519TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 09:39:515520 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5521 VideoEncoderConfig video_encoder_config;
5522 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5523 &video_encoder_config);
5524 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5525 vp9_settings.numberOfSpatialLayers = 3;
5526 // Since only one layer is active - automatic resize should be enabled.
5527 vp9_settings.automaticResizeOn = true;
5528 video_encoder_config.encoder_specific_settings =
5529 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5530 vp9_settings);
5531 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5532 video_encoder_config.content_type =
5533 VideoEncoderConfig::ContentType::kRealtimeVideo;
5534 // Simulcast layers are used to indicate which spatial layers are active.
5535 video_encoder_config.simulcast_layers.resize(3);
5536 video_encoder_config.simulcast_layers[0].active = false;
5537 video_encoder_config.simulcast_layers[1].active = true;
5538 video_encoder_config.simulcast_layers[2].active = false;
5539
5540 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5541 kMaxPayloadLength);
5542 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5543
5544 // The default bitrate limits for 360p should be used.
5545 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195546 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5547 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515548 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5549 EXPECT_FALSE(WaitForFrame(1000));
5550 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5551 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5552 VideoCodecType::kVideoCodecVP9);
5553 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5554 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5555 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5556 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5557 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5558 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5559 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5560 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5561
5562 // The default bitrate limits for 270p should be used.
5563 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 12:29:195564 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5565 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 09:39:515566 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5567 EXPECT_FALSE(WaitForFrame(1000));
5568 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5569 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5570 VideoCodecType::kVideoCodecVP9);
5571 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5572 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5573 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5574 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5575 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5576 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5577 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5578 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5579
5580 video_stream_encoder_->Stop();
5581}
5582
5583TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5584 webrtc::test::ScopedFieldTrials field_trials(
5585 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5586 VideoEncoderConfig video_encoder_config;
5587 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5588 &video_encoder_config);
5589 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5590 vp9_settings.numberOfSpatialLayers = 3;
5591 // Since only one layer is active - automatic resize should be enabled.
5592 vp9_settings.automaticResizeOn = true;
5593 video_encoder_config.encoder_specific_settings =
5594 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5595 vp9_settings);
5596 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5597 video_encoder_config.content_type =
5598 VideoEncoderConfig::ContentType::kRealtimeVideo;
5599 // Simulcast layers are used to indicate which spatial layers are active.
5600 video_encoder_config.simulcast_layers.resize(3);
5601 video_encoder_config.simulcast_layers[0].active = false;
5602 video_encoder_config.simulcast_layers[1].active = true;
5603 video_encoder_config.simulcast_layers[2].active = false;
5604
5605 // Reset encoder for field trials to take effect.
5606 ConfigureEncoder(video_encoder_config.Copy());
5607
5608 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5609 kMaxPayloadLength);
5610 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5611
5612 // The default bitrate limits for 360p should not be used.
5613 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 12:29:195614 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5615 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 09:39:515616 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5617 EXPECT_FALSE(WaitForFrame(1000));
5618 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 12:29:195619 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 09:39:515620 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5621 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5622 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5623 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5624 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5625 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5626
5627 video_stream_encoder_->Stop();
5628}
5629
5630TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5631 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5632 /*num_spatial_layers=*/1, /*screenshare=*/false);
5633
5634 // The default singlecast bitrate limits for 720p should not be used.
5635 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 12:29:195636 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5637 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 09:39:515638 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5639 EXPECT_FALSE(WaitForFrame(1000));
5640 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5641 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5642 VideoCodecType::kVideoCodecVP9);
5643 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5644 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5645 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5646 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5647 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5648 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5649
5650 video_stream_encoder_->Stop();
5651}
5652
5653TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 08:20:055654 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5655 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5656 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5657 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5658 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5659 fake_encoder_.SetResolutionBitrateLimits(
5660 {kEncoderLimits180p, kEncoderLimits720p});
5661
5662 VideoEncoderConfig video_encoder_config;
5663 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5664 &video_encoder_config);
5665 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5666 vp9_settings.numberOfSpatialLayers = 3;
5667 // Since only one layer is active - automatic resize should be enabled.
5668 vp9_settings.automaticResizeOn = true;
5669 video_encoder_config.encoder_specific_settings =
5670 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5671 vp9_settings);
5672 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5673 video_encoder_config.content_type =
5674 VideoEncoderConfig::ContentType::kRealtimeVideo;
5675 // Simulcast layers are used to indicate which spatial layers are active.
5676 video_encoder_config.simulcast_layers.resize(3);
5677 video_encoder_config.simulcast_layers[0].active = true;
5678 video_encoder_config.simulcast_layers[1].active = false;
5679 video_encoder_config.simulcast_layers[2].active = false;
5680
5681 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5682 kMaxPayloadLength);
5683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5684
5685 // Limits not applied on lowest stream, limits for 180p should not be used.
5686 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5687 EXPECT_FALSE(WaitForFrame(1000));
5688 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5689 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5690 VideoCodecType::kVideoCodecVP9);
5691 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5692 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5693 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5694 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5695 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5696 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5697 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5698 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5699
5700 video_stream_encoder_->Stop();
5701}
5702
5703TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 15:58:475704 InitialFrameDropActivatesWhenResolutionIncreases) {
5705 const int kWidth = 640;
5706 const int kHeight = 360;
5707
5708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5709 DataRate::BitsPerSec(kTargetBitrateBps),
5710 DataRate::BitsPerSec(kTargetBitrateBps),
5711 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5712 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5713 // Frame should not be dropped.
5714 WaitForEncodedFrame(1);
5715
5716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5717 DataRate::BitsPerSec(kLowTargetBitrateBps),
5718 DataRate::BitsPerSec(kLowTargetBitrateBps),
5719 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5720 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5721 // Frame should not be dropped, bitrate not too low for frame.
5722 WaitForEncodedFrame(2);
5723
5724 // Incoming resolution increases.
5725 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5726 // Expect to drop this frame, bitrate too low for frame.
5727 ExpectDroppedFrame();
5728
5729 // Expect the sink_wants to specify a scaled frame.
5730 EXPECT_TRUE_WAIT(
5731 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5732 video_stream_encoder_->Stop();
5733}
5734
5735TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5736 const int kWidth = 640;
5737 const int kHeight = 360;
5738 // So that quality scaling doesn't happen by itself.
5739 fake_encoder_.SetQp(kQpHigh);
5740
5741 AdaptingFrameForwarder source(&time_controller_);
5742 source.set_adaptation_enabled(true);
5743 video_stream_encoder_->SetSource(
5744 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5745
5746 int timestamp = 1;
5747
5748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5749 DataRate::BitsPerSec(kTargetBitrateBps),
5750 DataRate::BitsPerSec(kTargetBitrateBps),
5751 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5752 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5753 WaitForEncodedFrame(timestamp);
5754 timestamp += 9000;
5755 // Long pause to disable all first BWE drop logic.
5756 AdvanceTime(TimeDelta::Millis(1000));
5757
5758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5759 DataRate::BitsPerSec(kLowTargetBitrateBps),
5760 DataRate::BitsPerSec(kLowTargetBitrateBps),
5761 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5762 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5763 // Not dropped frame, as initial frame drop is disabled by now.
5764 WaitForEncodedFrame(timestamp);
5765 timestamp += 9000;
5766 AdvanceTime(TimeDelta::Millis(100));
5767
5768 // Quality adaptation down.
5769 video_stream_encoder_->TriggerQualityLow();
5770
5771 // Adaptation has an effect.
5772 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5773 5000);
5774
5775 // Frame isn't dropped as initial frame dropper is disabled.
5776 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5777 WaitForEncodedFrame(timestamp);
5778 timestamp += 9000;
5779 AdvanceTime(TimeDelta::Millis(100));
5780
5781 // Quality adaptation up.
5782 video_stream_encoder_->TriggerQualityHigh();
5783
5784 // Adaptation has an effect.
5785 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5786 5000);
5787
5788 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5789 // Frame should not be dropped, as initial framedropper is off.
5790 WaitForEncodedFrame(timestamp);
5791
5792 video_stream_encoder_->Stop();
5793}
5794
Åsa Persson7f354f82021-02-04 14:52:155795TEST_F(VideoStreamEncoderTest,
5796 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5797 const int kMinStartBps360p = 222000;
5798 fake_encoder_.SetResolutionBitrateLimits(
5799 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5800 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5801 800000)});
5802
5803 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5804 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5805 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5806 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5807 0, 0, 0);
5808 // Frame should not be dropped, bitrate not too low for frame.
5809 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5810 WaitForEncodedFrame(1);
5811
5812 // Incoming resolution increases, initial frame drop activates.
5813 // Frame should be dropped, link allocation too low for frame.
5814 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5815 ExpectDroppedFrame();
5816
5817 // Expect sink_wants to specify a scaled frame.
5818 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5819 5000);
5820 video_stream_encoder_->Stop();
5821}
5822
5823TEST_F(VideoStreamEncoderTest,
5824 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5825 const int kMinStartBps360p = 222000;
5826 fake_encoder_.SetResolutionBitrateLimits(
5827 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5828 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5829 800000)});
5830
5831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5832 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5833 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5834 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5835 0, 0, 0);
5836 // Frame should not be dropped, bitrate not too low for frame.
5837 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5838 WaitForEncodedFrame(1);
5839
5840 // Incoming resolution increases, initial frame drop activates.
5841 // Frame should be dropped, link allocation not too low for frame.
5842 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5843 WaitForEncodedFrame(2);
5844
5845 video_stream_encoder_->Stop();
5846}
5847
Åsa Perssone644a032019-11-08 14:56:005848TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5849 webrtc::test::ScopedFieldTrials field_trials(
5850 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5851
5852 // Reset encoder for field trials to take effect.
5853 VideoEncoderConfig config = video_encoder_config_.Copy();
5854 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 09:34:325855 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 14:56:005856 ConfigureEncoder(std::move(config));
5857 fake_encoder_.SetQp(kQpLow);
5858
5859 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 12:31:235860 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 14:56:005861 source.set_adaptation_enabled(true);
5862 video_stream_encoder_->SetSource(&source,
5863 DegradationPreference::MAINTAIN_FRAMERATE);
5864
5865 // Start at low bitrate.
5866 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 16:49:075867 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5868 DataRate::BitsPerSec(kLowBitrateBps),
5869 DataRate::BitsPerSec(kLowBitrateBps),
5870 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:005871
5872 // Expect first frame to be dropped and resolution to be limited.
5873 const int kWidth = 1280;
5874 const int kHeight = 720;
5875 const int64_t kFrameIntervalMs = 100;
5876 int64_t timestamp_ms = kFrameIntervalMs;
5877 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5878 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 14:29:095879 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5880 5000);
Åsa Perssone644a032019-11-08 14:56:005881
5882 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 16:49:075883 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5884 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:005885
5886 // Insert frames and advance |min_duration_ms|.
5887 for (size_t i = 1; i <= 10; i++) {
5888 timestamp_ms += kFrameIntervalMs;
5889 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5890 WaitForEncodedFrame(timestamp_ms);
5891 }
5892 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5893 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5894
Tomas Gunnarsson612445e2020-09-21 12:31:235895 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 14:56:005896
5897 // Insert frame should trigger high BW and release quality limitation.
5898 timestamp_ms += kFrameIntervalMs;
5899 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5900 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 16:49:075901 // The ramp-up code involves the adaptation queue, give it time to execute.
5902 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 11:12:125903 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 12:08:395904 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 14:56:005905
5906 // Frame should not be adapted.
5907 timestamp_ms += kFrameIntervalMs;
5908 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5909 WaitForEncodedFrame(kWidth, kHeight);
5910 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5911
5912 video_stream_encoder_->Stop();
5913}
5914
mflodmancc3d4422017-08-03 15:27:515915TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 11:12:285916 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 16:19:315917 webrtc::test::ScopedFieldTrials field_trials(
5918 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 12:31:235919 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 11:12:285920 source.set_adaptation_enabled(true);
5921 video_stream_encoder_->SetSource(&source,
5922 DegradationPreference::MAINTAIN_FRAMERATE);
5923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5924 DataRate::BitsPerSec(kTargetBitrateBps),
5925 DataRate::BitsPerSec(kTargetBitrateBps),
5926 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5927 fake_encoder_.SetQp(kQpHigh + 1);
5928 const int kWidth = 1280;
5929 const int kHeight = 720;
5930 const int64_t kFrameIntervalMs = 100;
5931 int64_t timestamp_ms = kFrameIntervalMs;
5932 for (size_t i = 1; i <= 100; i++) {
5933 timestamp_ms += kFrameIntervalMs;
5934 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5935 WaitForEncodedFrame(timestamp_ms);
5936 }
5937 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5938 // for the first time.
5939 // TODO(eshr): We should avoid these waits by using threads with simulated
5940 // time.
5941 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5942 2000 * 2.5 * 2);
5943 timestamp_ms += kFrameIntervalMs;
5944 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5945 WaitForEncodedFrame(timestamp_ms);
5946 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5947 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5948 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5949
5950 // Disable Quality scaling by turning off scaler on the encoder and
5951 // reconfiguring.
5952 fake_encoder_.SetQualityScaling(false);
5953 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5954 kMaxPayloadLength);
5955 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 12:31:235956 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 11:12:285957 // Since we turned off the quality scaler, the adaptations made by it are
5958 // removed.
5959 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5960 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5961
5962 video_stream_encoder_->Stop();
5963}
5964
5965TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:315966 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5967 const int kTooSmallWidth = 10;
5968 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 16:49:075969 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075970 DataRate::BitsPerSec(kTargetBitrateBps),
5971 DataRate::BitsPerSec(kTargetBitrateBps),
5972 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:315973
Taylor Brandstetter49fcc102018-05-16 21:20:415974 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:315975 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:515976 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:415977 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 12:08:395978 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 08:47:315979 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5980
5981 // Trigger adapt down, too small frame, expect no change.
5982 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:525983 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:515984 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 12:08:395985 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 08:47:315986 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5987 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5988
mflodmancc3d4422017-08-03 15:27:515989 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:315990}
5991
mflodmancc3d4422017-08-03 15:27:515992TEST_F(VideoStreamEncoderTest,
5993 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:225994 const int kTooSmallWidth = 10;
5995 const int kTooSmallHeight = 10;
5996 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 16:49:075997 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:075998 DataRate::BitsPerSec(kTargetBitrateBps),
5999 DataRate::BitsPerSec(kTargetBitrateBps),
6000 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226001
Taylor Brandstetter49fcc102018-05-16 21:20:416002 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:226003 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:416004 video_stream_encoder_->SetSource(&source,
6005 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:396006 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-14 06:25:226007 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6008 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6009
6010 // Trigger adapt down, expect limited framerate.
6011 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526012 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:516013 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396014 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226015 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6016 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6017 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6018
6019 // Trigger adapt down, too small frame, expect no change.
6020 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:526021 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:516022 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 12:08:396023 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6025 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6026 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6027
mflodmancc3d4422017-08-03 15:27:516028 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226029}
6030
mflodmancc3d4422017-08-03 15:27:516031TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 08:12:526032 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 16:49:076033 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076034 DataRate::BitsPerSec(kTargetBitrateBps),
6035 DataRate::BitsPerSec(kTargetBitrateBps),
6036 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 07:51:476037 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 08:12:526038 const int kFrameWidth = 1280;
6039 const int kFrameHeight = 720;
6040 video_source_.IncomingCapturedFrame(
6041 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526042 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 15:27:516043 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:526044}
6045
sprangb1ca0732017-02-01 16:38:126046// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 15:27:516047TEST_F(VideoStreamEncoderTest,
6048 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 16:49:076049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076050 DataRate::BitsPerSec(kTargetBitrateBps),
6051 DataRate::BitsPerSec(kTargetBitrateBps),
6052 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 16:38:126053
6054 const int kFrameWidth = 1280;
6055 const int kFrameHeight = 720;
6056 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 15:27:516057 // requested by
6058 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 16:38:126059 video_source_.set_adaptation_enabled(true);
6060
6061 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196062 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526063 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126064
6065 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516066 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 16:38:126067 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196068 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526069 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 16:38:126070
asaperssonfab67072017-04-04 12:51:496071 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 10:24:336072 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 16:38:126073 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:196074 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526075 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:126076
mflodmancc3d4422017-08-03 15:27:516077 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 16:38:126078}
sprangfe627f32017-03-29 15:24:596079
mflodmancc3d4422017-08-03 15:27:516080TEST_F(VideoStreamEncoderTest,
6081 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-03 06:53:046082 const int kFrameWidth = 1280;
6083 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:046084
Henrik Boström381d1092020-05-12 16:49:076085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076086 DataRate::BitsPerSec(kTargetBitrateBps),
6087 DataRate::BitsPerSec(kTargetBitrateBps),
6088 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516089 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416090 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046091 video_source_.set_adaptation_enabled(true);
6092
Tomas Gunnarsson612445e2020-09-21 12:31:236093 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046094
6095 video_source_.IncomingCapturedFrame(
6096 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526097 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046098
6099 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 15:27:516100 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046101
6102 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 14:06:526103 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046104 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046105 video_source_.IncomingCapturedFrame(
6106 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526107 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:046108 }
6109
6110 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 15:27:516111 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046112 int num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526113 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046114 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046115 video_source_.IncomingCapturedFrame(
6116 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526117 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046118 ++num_frames_dropped;
6119 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566120 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046121 }
6122 }
6123
sprang4847ae62017-06-27 14:06:526124 // Add some slack to account for frames dropped by the frame dropper.
6125 const int kErrorMargin = 1;
6126 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046127 kErrorMargin);
6128
6129 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 15:27:516130 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:046131 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 08:42:196132 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046133 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046134 video_source_.IncomingCapturedFrame(
6135 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526136 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046137 ++num_frames_dropped;
6138 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566139 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046140 }
6141 }
sprang4847ae62017-06-27 14:06:526142 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-03 06:53:046143 kErrorMargin);
6144
6145 // Go back up one step.
Henrik Boström91aa7322020-04-28 10:24:336146 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046147 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526148 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046149 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046150 video_source_.IncomingCapturedFrame(
6151 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526152 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046153 ++num_frames_dropped;
6154 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566155 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046156 }
6157 }
sprang4847ae62017-06-27 14:06:526158 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:046159 kErrorMargin);
6160
6161 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 10:24:336162 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-03 06:53:046163 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:526164 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:046165 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:046166 video_source_.IncomingCapturedFrame(
6167 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526168 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:046169 ++num_frames_dropped;
6170 } else {
Åsa Perssonc74d8da2017-12-04 13:13:566171 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:046172 }
6173 }
6174 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6175
mflodmancc3d4422017-08-03 15:27:516176 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046177}
6178
mflodmancc3d4422017-08-03 15:27:516179TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-03 06:53:046180 const int kFramerateFps = 5;
6181 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-03 06:53:046182 const int kFrameWidth = 1280;
6183 const int kFrameHeight = 720;
6184
sprang4847ae62017-06-27 14:06:526185 // Reconfigure encoder with two temporal layers and screensharing, which will
6186 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 07:51:476187 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 14:06:526188
Henrik Boström381d1092020-05-12 16:49:076189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076190 DataRate::BitsPerSec(kTargetBitrateBps),
6191 DataRate::BitsPerSec(kTargetBitrateBps),
6192 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516193 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:416194 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:046195 video_source_.set_adaptation_enabled(true);
6196
Tomas Gunnarsson612445e2020-09-21 12:31:236197 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-03 06:53:046198
6199 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-09 01:04:296200 rtc::VideoSinkWants last_wants;
6201 do {
6202 last_wants = video_source_.sink_wants();
6203
sprangc5d62e22017-04-03 06:53:046204 // Insert frames to get a new fps estimate...
6205 for (int j = 0; j < kFramerateFps; ++j) {
6206 video_source_.IncomingCapturedFrame(
6207 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-09 01:04:296208 if (video_source_.last_sent_width()) {
6209 sink_.WaitForEncodedFrame(timestamp_ms);
6210 }
sprangc5d62e22017-04-03 06:53:046211 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:236212 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-03 06:53:046213 }
6214 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 15:27:516215 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:296216 } while (video_source_.sink_wants().max_framerate_fps <
6217 last_wants.max_framerate_fps);
sprangc5d62e22017-04-03 06:53:046218
Evan Shrubsole5cd7eb82020-05-25 12:08:396219 EXPECT_THAT(video_source_.sink_wants(),
6220 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-14 06:25:226221
mflodmancc3d4422017-08-03 15:27:516222 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:046223}
asaperssonf7e294d2017-06-14 06:25:226224
mflodmancc3d4422017-08-03 15:27:516225TEST_F(VideoStreamEncoderTest,
6226 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:226227 const int kWidth = 1280;
6228 const int kHeight = 720;
6229 const int64_t kFrameIntervalMs = 150;
6230 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076231 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076232 DataRate::BitsPerSec(kTargetBitrateBps),
6233 DataRate::BitsPerSec(kTargetBitrateBps),
6234 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226235
Taylor Brandstetter49fcc102018-05-16 21:20:416236 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236237 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226238 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416239 video_stream_encoder_->SetSource(&source,
6240 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226241 timestamp_ms += kFrameIntervalMs;
6242 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526243 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396244 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226245 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6246 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6247 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6248
6249 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516250 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226251 timestamp_ms += kFrameIntervalMs;
6252 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526253 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396254 EXPECT_THAT(source.sink_wants(),
6255 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226256 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6257 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6258 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6259
6260 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516261 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226262 timestamp_ms += kFrameIntervalMs;
6263 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526264 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546265 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226266 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6267 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6268 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6269
6270 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516271 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226272 timestamp_ms += kFrameIntervalMs;
6273 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526274 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546275 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226276 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6277 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6278 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6279
6280 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516281 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226282 timestamp_ms += kFrameIntervalMs;
6283 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526284 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546285 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6287 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6288 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6289
6290 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516291 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226292 timestamp_ms += kFrameIntervalMs;
6293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526294 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546295 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226296 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6297 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6298 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6299
6300 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516301 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226302 timestamp_ms += kFrameIntervalMs;
6303 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526304 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546305 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226306 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6307 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6308 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6309
6310 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 15:27:516311 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226312 timestamp_ms += kFrameIntervalMs;
6313 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526314 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546315 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226316 rtc::VideoSinkWants last_wants = source.sink_wants();
6317 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6318 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6319 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6320
6321 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 15:27:516322 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226323 timestamp_ms += kFrameIntervalMs;
6324 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526325 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546326 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-14 06:25:226327 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6329 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6330
Evan Shrubsole64469032020-06-11 08:45:296331 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:516332 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226333 timestamp_ms += kFrameIntervalMs;
6334 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526335 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546336 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226337 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6338 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6339 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6340
6341 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:516342 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226343 timestamp_ms += kFrameIntervalMs;
6344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526345 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546346 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226347 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6348 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6349 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6350
6351 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516352 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226353 timestamp_ms += kFrameIntervalMs;
6354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526355 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546356 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226357 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6358 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6359 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6360
6361 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516362 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226363 timestamp_ms += kFrameIntervalMs;
6364 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526365 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546366 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226367 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6368 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6369 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6370
6371 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516372 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226373 timestamp_ms += kFrameIntervalMs;
6374 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526375 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546376 EXPECT_THAT(source.sink_wants(), FpsMax());
6377 EXPECT_EQ(source.sink_wants().max_pixel_count,
6378 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-14 06:25:226379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6380 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6381 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6382
6383 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516384 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226385 timestamp_ms += kFrameIntervalMs;
6386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526387 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546388 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6390 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6391 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6392
Åsa Persson30ab0152019-08-27 10:22:336393 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516394 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226395 timestamp_ms += kFrameIntervalMs;
6396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526397 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546398 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396399 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226400 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6401 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6402 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6403
6404 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516405 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396406 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226407 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6408
mflodmancc3d4422017-08-03 15:27:516409 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226410}
6411
mflodmancc3d4422017-08-03 15:27:516412TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-14 06:25:226413 const int kWidth = 1280;
6414 const int kHeight = 720;
6415 const int64_t kFrameIntervalMs = 150;
6416 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076418 DataRate::BitsPerSec(kTargetBitrateBps),
6419 DataRate::BitsPerSec(kTargetBitrateBps),
6420 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226421
Taylor Brandstetter49fcc102018-05-16 21:20:416422 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236423 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226424 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416425 video_stream_encoder_->SetSource(&source,
6426 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226427 timestamp_ms += kFrameIntervalMs;
6428 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526429 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396430 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226431 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6432 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6433 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6434 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6435 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6436 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6437
6438 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:516439 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226440 timestamp_ms += kFrameIntervalMs;
6441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526442 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396443 EXPECT_THAT(source.sink_wants(),
6444 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-14 06:25:226445 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6447 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6448 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6449 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6450 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6451
6452 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516453 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226454 timestamp_ms += kFrameIntervalMs;
6455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526456 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546457 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6459 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6460 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6461 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6462 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6463 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6464
6465 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516466 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226467 timestamp_ms += kFrameIntervalMs;
6468 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526469 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546470 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 08:45:296471 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-14 06:25:226472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6473 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6474 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6475 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6476 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6477
Evan Shrubsole64469032020-06-11 08:45:296478 // Trigger cpu adapt up, expect no change since QP is most limited.
6479 {
6480 // Store current sink wants since we expect no change and if there is no
6481 // change then last_wants() is not updated.
6482 auto previous_sink_wants = source.sink_wants();
6483 video_stream_encoder_->TriggerCpuUnderuse();
6484 timestamp_ms += kFrameIntervalMs;
6485 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6486 WaitForEncodedFrame(timestamp_ms);
6487 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6488 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6489 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6490 }
6491
6492 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6493 video_stream_encoder_->TriggerQualityHigh();
6494 timestamp_ms += kFrameIntervalMs;
6495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6496 WaitForEncodedFrame(timestamp_ms);
6497 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6500 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6501 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6502 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6503 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6504
6505 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6506 // expect increased resolution (960x540@30fps).
6507 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336508 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226509 timestamp_ms += kFrameIntervalMs;
6510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526511 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 08:45:296512 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226513 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6514 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6515 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6516 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6517 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296518 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226519
Evan Shrubsole64469032020-06-11 08:45:296520 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6521 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:516522 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 10:24:336523 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226524 timestamp_ms += kFrameIntervalMs;
6525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526526 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 14:19:546527 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 12:08:396528 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6531 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6532 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6533 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296534 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226535
6536 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516537 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396538 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226539 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296540 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226541
mflodmancc3d4422017-08-03 15:27:516542 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226543}
6544
mflodmancc3d4422017-08-03 15:27:516545TEST_F(VideoStreamEncoderTest,
6546 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-14 06:25:226547 const int kWidth = 640;
6548 const int kHeight = 360;
6549 const int kFpsLimit = 15;
6550 const int64_t kFrameIntervalMs = 150;
6551 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 16:49:076552 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076553 DataRate::BitsPerSec(kTargetBitrateBps),
6554 DataRate::BitsPerSec(kTargetBitrateBps),
6555 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:226556
Taylor Brandstetter49fcc102018-05-16 21:20:416557 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 12:31:236558 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-14 06:25:226559 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:416560 video_stream_encoder_->SetSource(&source,
6561 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:226562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526564 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 12:08:396565 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6567 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6569 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6570 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6571 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6572
6573 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:516574 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:226575 timestamp_ms += kFrameIntervalMs;
6576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526577 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396578 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-14 06:25:226579 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6580 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6581 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6582 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6583 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6584 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6585
6586 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:516587 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:226588 timestamp_ms += kFrameIntervalMs;
6589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526590 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546591 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:226594 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6595 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6596 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6597 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6598
Evan Shrubsole64469032020-06-11 08:45:296599 // Trigger cpu adapt up, expect no change because quality is most limited.
6600 {
6601 auto previous_sink_wants = source.sink_wants();
6602 // Store current sink wants since we expect no change ind if there is no
6603 // change then last__wants() is not updated.
6604 video_stream_encoder_->TriggerCpuUnderuse();
6605 timestamp_ms += kFrameIntervalMs;
6606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6607 WaitForEncodedFrame(timestamp_ms);
6608 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6609 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6610 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6611 }
6612
6613 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6614 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:226615 timestamp_ms += kFrameIntervalMs;
6616 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526617 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 14:19:546618 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-14 06:25:226619 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6620 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6621 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 08:45:296622 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6623 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6624 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226625
Evan Shrubsole64469032020-06-11 08:45:296626 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:516627 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 08:45:296628 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-14 06:25:226629 timestamp_ms += kFrameIntervalMs;
6630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:526631 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 12:08:396632 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226633 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6634 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6635 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6636 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6637 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296638 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226639
6640 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:516641 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 12:08:396642 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-14 06:25:226643 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 08:45:296644 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:226645
mflodmancc3d4422017-08-03 15:27:516646 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:226647}
6648
mflodmancc3d4422017-08-03 15:27:516649TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 13:53:486650 const int kFrameWidth = 1920;
6651 const int kFrameHeight = 1080;
6652 // 3/4 of 1920.
6653 const int kAdaptedFrameWidth = 1440;
6654 // 3/4 of 1080 rounded down to multiple of 4.
6655 const int kAdaptedFrameHeight = 808;
6656 const int kFramerate = 24;
6657
Henrik Boström381d1092020-05-12 16:49:076658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076659 DataRate::BitsPerSec(kTargetBitrateBps),
6660 DataRate::BitsPerSec(kTargetBitrateBps),
6661 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 13:53:486662 // Trigger reconfigure encoder (without resetting the entire instance).
6663 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 10:57:586664 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6665 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 13:53:486666 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 13:53:486667 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 10:57:586668 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 15:27:516669 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:476670 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:516671 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 13:53:486672
6673 video_source_.set_adaptation_enabled(true);
6674
6675 video_source_.IncomingCapturedFrame(
6676 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526677 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 13:53:486678
6679 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:516680 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 13:53:486681 video_source_.IncomingCapturedFrame(
6682 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:526683 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 13:53:486684
mflodmancc3d4422017-08-03 15:27:516685 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 13:53:486686}
6687
mflodmancc3d4422017-08-03 15:27:516688TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 14:06:526689 const int kFrameWidth = 1280;
6690 const int kFrameHeight = 720;
6691 const int kLowFps = 2;
6692 const int kHighFps = 30;
6693
Henrik Boström381d1092020-05-12 16:49:076694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076695 DataRate::BitsPerSec(kTargetBitrateBps),
6696 DataRate::BitsPerSec(kTargetBitrateBps),
6697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:526698
Tomas Gunnarsson612445e2020-09-21 12:31:236699 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:526700 max_framerate_ = kLowFps;
6701
6702 // Insert 2 seconds of 2fps video.
6703 for (int i = 0; i < kLowFps * 2; ++i) {
6704 video_source_.IncomingCapturedFrame(
6705 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6706 WaitForEncodedFrame(timestamp_ms);
6707 timestamp_ms += 1000 / kLowFps;
6708 }
6709
6710 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 16:49:076711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076712 DataRate::BitsPerSec(kTargetBitrateBps),
6713 DataRate::BitsPerSec(kTargetBitrateBps),
6714 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:526715 video_source_.IncomingCapturedFrame(
6716 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6717 WaitForEncodedFrame(timestamp_ms);
6718 timestamp_ms += 1000 / kLowFps;
6719
6720 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6721
6722 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 08:48:486723 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 14:06:526724 const int kFrameIntervalMs = 1000 / kHighFps;
6725 max_framerate_ = kHighFps;
6726 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6727 video_source_.IncomingCapturedFrame(
6728 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6729 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6730 // be dropped if the encoder hans't been updated with the new higher target
6731 // framerate yet, causing it to overshoot the target bitrate and then
6732 // suffering the wrath of the media optimizer.
6733 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6734 timestamp_ms += kFrameIntervalMs;
6735 }
6736
6737 // Don expect correct measurement just yet, but it should be higher than
6738 // before.
6739 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6740
mflodmancc3d4422017-08-03 15:27:516741 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:526742}
6743
mflodmancc3d4422017-08-03 15:27:516744TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 14:06:526745 const int kFrameWidth = 1280;
6746 const int kFrameHeight = 720;
6747 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 13:09:056748 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 09:26:036749 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 13:09:056750 kVideoBitrateAllocation);
sprang4847ae62017-06-27 14:06:526751
Henrik Boström381d1092020-05-12 16:49:076752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076753 DataRate::BitsPerSec(kTargetBitrateBps),
6754 DataRate::BitsPerSec(kTargetBitrateBps),
6755 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:516756 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 14:06:526757
6758 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 12:31:236759 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 14:06:526760 video_source_.IncomingCapturedFrame(
6761 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6762 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 13:09:056763 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:526764
6765 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 16:49:076766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6767 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6768 0, 1, 0);
sprang4847ae62017-06-27 14:06:526769
6770 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 08:48:486771 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 12:31:236772 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 14:06:526773
Per Kjellanderdcef6412020-10-07 13:09:056774 // No more allocations has been made.
sprang4847ae62017-06-27 14:06:526775 video_source_.IncomingCapturedFrame(
6776 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6777 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 13:09:056778 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 14:06:526779
mflodmancc3d4422017-08-03 15:27:516780 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:526781}
ilnik6b826ef2017-06-16 13:53:486782
Niels Möller4db138e2018-04-19 07:04:136783TEST_F(VideoStreamEncoderTest,
6784 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6785 const int kFrameWidth = 1280;
6786 const int kFrameHeight = 720;
6787 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 16:49:076788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076789 DataRate::BitsPerSec(kTargetBitrateBps),
6790 DataRate::BitsPerSec(kTargetBitrateBps),
6791 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:136792 video_source_.IncomingCapturedFrame(
6793 CreateFrame(1, kFrameWidth, kFrameHeight));
6794 WaitForEncodedFrame(1);
6795 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6796 .low_encode_usage_threshold_percent,
6797 default_options.low_encode_usage_threshold_percent);
6798 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6799 .high_encode_usage_threshold_percent,
6800 default_options.high_encode_usage_threshold_percent);
6801 video_stream_encoder_->Stop();
6802}
6803
6804TEST_F(VideoStreamEncoderTest,
6805 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6806 const int kFrameWidth = 1280;
6807 const int kFrameHeight = 720;
6808 CpuOveruseOptions hardware_options;
6809 hardware_options.low_encode_usage_threshold_percent = 150;
6810 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 11:42:186811 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 07:04:136812
Henrik Boström381d1092020-05-12 16:49:076813 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076814 DataRate::BitsPerSec(kTargetBitrateBps),
6815 DataRate::BitsPerSec(kTargetBitrateBps),
6816 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:136817 video_source_.IncomingCapturedFrame(
6818 CreateFrame(1, kFrameWidth, kFrameHeight));
6819 WaitForEncodedFrame(1);
6820 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6821 .low_encode_usage_threshold_percent,
6822 hardware_options.low_encode_usage_threshold_percent);
6823 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6824 .high_encode_usage_threshold_percent,
6825 hardware_options.high_encode_usage_threshold_percent);
6826 video_stream_encoder_->Stop();
6827}
6828
Jakob Ivarsson461b1d92021-01-22 15:27:436829TEST_F(VideoStreamEncoderTest,
6830 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6831 const int kFrameWidth = 1280;
6832 const int kFrameHeight = 720;
6833
6834 const CpuOveruseOptions default_options;
6835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6836 DataRate::BitsPerSec(kTargetBitrateBps),
6837 DataRate::BitsPerSec(kTargetBitrateBps),
6838 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6839 video_source_.IncomingCapturedFrame(
6840 CreateFrame(1, kFrameWidth, kFrameHeight));
6841 WaitForEncodedFrame(1);
6842 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6843 .low_encode_usage_threshold_percent,
6844 default_options.low_encode_usage_threshold_percent);
6845 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6846 .high_encode_usage_threshold_percent,
6847 default_options.high_encode_usage_threshold_percent);
6848
6849 CpuOveruseOptions hardware_options;
6850 hardware_options.low_encode_usage_threshold_percent = 150;
6851 hardware_options.high_encode_usage_threshold_percent = 200;
6852 fake_encoder_.SetIsHardwareAccelerated(true);
6853
6854 video_source_.IncomingCapturedFrame(
6855 CreateFrame(2, kFrameWidth, kFrameHeight));
6856 WaitForEncodedFrame(2);
6857
6858 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6859 .low_encode_usage_threshold_percent,
6860 hardware_options.low_encode_usage_threshold_percent);
6861 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6862 .high_encode_usage_threshold_percent,
6863 hardware_options.high_encode_usage_threshold_percent);
6864
6865 video_stream_encoder_->Stop();
6866}
6867
Niels Möller6bb5ab92019-01-11 10:11:106868TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6869 const int kFrameWidth = 320;
6870 const int kFrameHeight = 240;
6871 const int kFps = 30;
6872 const int kTargetBitrateBps = 120000;
6873 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6874
Henrik Boström381d1092020-05-12 16:49:076875 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076876 DataRate::BitsPerSec(kTargetBitrateBps),
6877 DataRate::BitsPerSec(kTargetBitrateBps),
6878 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:106879
Tomas Gunnarsson612445e2020-09-21 12:31:236880 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:106881 max_framerate_ = kFps;
6882
6883 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6884 fake_encoder_.SimulateOvershoot(1.0);
6885 int num_dropped = 0;
6886 for (int i = 0; i < kNumFramesInRun; ++i) {
6887 video_source_.IncomingCapturedFrame(
6888 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6889 // Wait up to two frame durations for a frame to arrive.
6890 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6891 ++num_dropped;
6892 }
6893 timestamp_ms += 1000 / kFps;
6894 }
6895
Erik Språnga8d48ab2019-02-08 13:17:406896 // Framerate should be measured to be near the expected target rate.
6897 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6898
6899 // Frame drops should be within 5% of expected 0%.
6900 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:106901
6902 // Make encoder produce frames at double the expected bitrate during 3 seconds
6903 // of video, verify number of drops. Rate needs to be slightly changed in
6904 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 15:20:176905 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 15:44:426906 const RateControlSettings trials =
6907 RateControlSettings::ParseFromFieldTrials();
6908 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 15:20:176909 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 15:44:426910 // frame dropping since the adjuter will try to just lower the target
6911 // bitrate rather than drop frames. If network headroom can be used, it
6912 // doesn't push back as hard so we don't need quite as much overshoot.
6913 // These numbers are unfortunately a bit magical but there's not trivial
6914 // way to algebraically infer them.
6915 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6916 overshoot_factor = 2.4;
6917 } else {
6918 overshoot_factor = 4.0;
6919 }
Erik Språng7ca375c2019-02-06 15:20:176920 }
6921 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 16:49:076922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076923 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6924 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6925 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:106926 num_dropped = 0;
6927 for (int i = 0; i < kNumFramesInRun; ++i) {
6928 video_source_.IncomingCapturedFrame(
6929 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6930 // Wait up to two frame durations for a frame to arrive.
6931 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6932 ++num_dropped;
6933 }
6934 timestamp_ms += 1000 / kFps;
6935 }
6936
Henrik Boström381d1092020-05-12 16:49:076937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076938 DataRate::BitsPerSec(kTargetBitrateBps),
6939 DataRate::BitsPerSec(kTargetBitrateBps),
6940 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 13:17:406941
6942 // Target framerate should be still be near the expected target, despite
6943 // the frame drops.
6944 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6945
6946 // Frame drops should be within 5% of expected 50%.
6947 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:106948
6949 video_stream_encoder_->Stop();
6950}
6951
6952TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6953 const int kFrameWidth = 320;
6954 const int kFrameHeight = 240;
6955 const int kActualInputFps = 24;
6956 const int kTargetBitrateBps = 120000;
6957
6958 ASSERT_GT(max_framerate_, kActualInputFps);
6959
Tomas Gunnarsson612445e2020-09-21 12:31:236960 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 10:11:106961 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 16:49:076962 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076963 DataRate::BitsPerSec(kTargetBitrateBps),
6964 DataRate::BitsPerSec(kTargetBitrateBps),
6965 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:106966
6967 // Insert 3 seconds of video, with an input fps lower than configured max.
6968 for (int i = 0; i < kActualInputFps * 3; ++i) {
6969 video_source_.IncomingCapturedFrame(
6970 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6971 // Wait up to two frame durations for a frame to arrive.
6972 WaitForEncodedFrame(timestamp_ms);
6973 timestamp_ms += 1000 / kActualInputFps;
6974 }
6975
6976 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6977
6978 video_stream_encoder_->Stop();
6979}
6980
Tomas Gunnarsson612445e2020-09-21 12:31:236981TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:266982 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 16:49:076983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:076984 DataRate::BitsPerSec(kTargetBitrateBps),
6985 DataRate::BitsPerSec(kTargetBitrateBps),
6986 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:266987
6988 fake_encoder_.BlockNextEncode();
6989 video_source_.IncomingCapturedFrame(
6990 CreateFrameWithUpdatedPixel(1, nullptr, 0));
6991 WaitForEncodedFrame(1);
6992 // On the very first frame full update should be forced.
6993 rect = fake_encoder_.GetLastUpdateRect();
6994 EXPECT_EQ(rect.offset_x, 0);
6995 EXPECT_EQ(rect.offset_y, 0);
6996 EXPECT_EQ(rect.height, codec_height_);
6997 EXPECT_EQ(rect.width, codec_width_);
6998 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
6999 // call to ContinueEncode.
7000 video_source_.IncomingCapturedFrame(
7001 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7002 ExpectDroppedFrame();
7003 video_source_.IncomingCapturedFrame(
7004 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7005 ExpectDroppedFrame();
7006 fake_encoder_.ContinueEncode();
7007 WaitForEncodedFrame(3);
7008 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7009 rect = fake_encoder_.GetLastUpdateRect();
7010 EXPECT_EQ(rect.offset_x, 1);
7011 EXPECT_EQ(rect.offset_y, 0);
7012 EXPECT_EQ(rect.width, 10);
7013 EXPECT_EQ(rect.height, 1);
7014
7015 video_source_.IncomingCapturedFrame(
7016 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7017 WaitForEncodedFrame(4);
7018 // Previous frame was encoded, so no accumulation should happen.
7019 rect = fake_encoder_.GetLastUpdateRect();
7020 EXPECT_EQ(rect.offset_x, 0);
7021 EXPECT_EQ(rect.offset_y, 0);
7022 EXPECT_EQ(rect.width, 1);
7023 EXPECT_EQ(rect.height, 1);
7024
7025 video_stream_encoder_->Stop();
7026}
7027
Erik Språngd7329ca2019-02-21 20:19:537028TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 16:49:077029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077030 DataRate::BitsPerSec(kTargetBitrateBps),
7031 DataRate::BitsPerSec(kTargetBitrateBps),
7032 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537033
7034 // First frame is always keyframe.
7035 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7036 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 14:43:587037 EXPECT_THAT(
7038 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127039 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537040
7041 // Insert delta frame.
7042 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7043 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 14:43:587044 EXPECT_THAT(
7045 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127046 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537047
7048 // Request next frame be a key-frame.
7049 video_stream_encoder_->SendKeyFrame();
7050 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7051 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 14:43:587052 EXPECT_THAT(
7053 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127054 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537055
7056 video_stream_encoder_->Stop();
7057}
7058
7059TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7060 // Setup simulcast with three streams.
7061 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077063 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7064 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7065 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537066 // Wait for all three layers before triggering event.
7067 sink_.SetNumExpectedLayers(3);
7068
7069 // First frame is always keyframe.
7070 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7071 WaitForEncodedFrame(1);
7072 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127073 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7074 VideoFrameType::kVideoFrameKey,
7075 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537076
7077 // Insert delta frame.
7078 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7079 WaitForEncodedFrame(2);
7080 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127081 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7082 VideoFrameType::kVideoFrameDelta,
7083 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537084
7085 // Request next frame be a key-frame.
7086 // Only first stream is configured to produce key-frame.
7087 video_stream_encoder_->SendKeyFrame();
7088 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7089 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 11:45:397090
7091 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7092 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 20:19:537093 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127094 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 11:45:397095 VideoFrameType::kVideoFrameKey,
7096 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537097
7098 video_stream_encoder_->Stop();
7099}
7100
7101TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7102 // Configure internal source factory and setup test again.
7103 encoder_factory_.SetHasInternalSource(true);
7104 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077105 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077106 DataRate::BitsPerSec(kTargetBitrateBps),
7107 DataRate::BitsPerSec(kTargetBitrateBps),
7108 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:537109
7110 // Call encoder directly, simulating internal source where encoded frame
7111 // callback in VideoStreamEncoder is called despite no OnFrame().
7112 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7113 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:587114 EXPECT_THAT(
7115 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127116 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537117
Niels Möller8f7ce222019-03-21 14:43:587118 const std::vector<VideoFrameType> kDeltaFrame = {
7119 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:537120 // Need to set timestamp manually since manually for injected frame.
7121 VideoFrame frame = CreateFrame(101, nullptr);
7122 frame.set_timestamp(101);
7123 fake_encoder_.InjectFrame(frame, false);
7124 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:587125 EXPECT_THAT(
7126 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127127 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:537128
7129 // Request key-frame. The forces a dummy frame down into the encoder.
7130 fake_encoder_.ExpectNullFrame();
7131 video_stream_encoder_->SendKeyFrame();
7132 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:587133 EXPECT_THAT(
7134 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:127135 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:537136
7137 video_stream_encoder_->Stop();
7138}
Erik Språngb7cb7b52019-02-26 14:52:337139
7140TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7141 // Configure internal source factory and setup test again.
7142 encoder_factory_.SetHasInternalSource(true);
7143 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 16:49:077144 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077145 DataRate::BitsPerSec(kTargetBitrateBps),
7146 DataRate::BitsPerSec(kTargetBitrateBps),
7147 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 14:52:337148
7149 int64_t timestamp = 1;
7150 EncodedImage image;
Erik Språngb7cb7b52019-02-26 14:52:337151 image.capture_time_ms_ = ++timestamp;
7152 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7153 const int64_t kEncodeFinishDelayMs = 10;
7154 image.timing_.encode_start_ms = timestamp;
7155 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 09:12:577156 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 14:52:337157 // Wait for frame without incrementing clock.
7158 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7159 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7160 // capture timestamp should be kEncodeFinishDelayMs in the past.
7161 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 12:31:237162 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 14:52:337163
7164 video_stream_encoder_->Stop();
7165}
Mirta Dvornicic28f0eb22019-05-28 14:30:167166
7167TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237168 // SPS contains VUI with restrictions on the maximum number of reordered
7169 // pictures, there is no need to rewrite the bitstream to enable faster
7170 // decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167171 ResetEncoder("H264", 1, 1, 1, false);
7172
Mirta Dvornicic97910da2020-07-14 13:29:237173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7174 DataRate::BitsPerSec(kTargetBitrateBps),
7175 DataRate::BitsPerSec(kTargetBitrateBps),
7176 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7177 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167178
Mirta Dvornicic97910da2020-07-14 13:29:237179 fake_encoder_.SetEncodedImageData(
7180 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167181
Mirta Dvornicic97910da2020-07-14 13:29:237182 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7183 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167184
7185 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7186 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167187
7188 video_stream_encoder_->Stop();
7189}
7190
7191TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 13:29:237192 // SPS does not contain VUI, the bitstream is will be rewritten with added
7193 // VUI with restrictions on the maximum number of reordered pictures to
7194 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 14:30:167195 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7196 0x00, 0x00, 0x03, 0x03, 0xF4,
7197 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 14:30:167198 ResetEncoder("H264", 1, 1, 1, false);
7199
Mirta Dvornicic97910da2020-07-14 13:29:237200 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7201 DataRate::BitsPerSec(kTargetBitrateBps),
7202 DataRate::BitsPerSec(kTargetBitrateBps),
7203 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7204 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 14:30:167205
Mirta Dvornicic97910da2020-07-14 13:29:237206 fake_encoder_.SetEncodedImageData(
7207 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 14:30:167208
Mirta Dvornicic97910da2020-07-14 13:29:237209 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7210 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 14:30:167211
7212 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7213 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 14:30:167214
7215 video_stream_encoder_->Stop();
7216}
7217
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147218TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7219 const int kFrameWidth = 1280;
7220 const int kFrameHeight = 720;
7221 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7222
Henrik Boström381d1092020-05-12 16:49:077223 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077224 DataRate::BitsPerSec(kTargetBitrateBps),
7225 DataRate::BitsPerSec(kTargetBitrateBps),
7226 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147227 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7228
7229 // Insert a first video frame. It should be dropped because of downscale in
7230 // resolution.
Tomas Gunnarsson612445e2020-09-21 12:31:237231 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147232 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7233 frame.set_rotation(kVideoRotation_270);
7234 video_source_.IncomingCapturedFrame(frame);
7235
7236 ExpectDroppedFrame();
7237
7238 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237239 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147240 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7241 frame.set_rotation(kVideoRotation_90);
7242 video_source_.IncomingCapturedFrame(frame);
7243
7244 WaitForEncodedFrame(timestamp_ms);
7245 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7246
7247 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 12:31:237248 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:147249 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7250 frame.set_rotation(kVideoRotation_180);
7251 video_source_.IncomingCapturedFrame(frame);
7252
7253 WaitForEncodedFrame(timestamp_ms);
7254 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7255
7256 video_stream_encoder_->Stop();
7257}
7258
Erik Språng5056af02019-09-02 13:53:117259TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7260 const int kFrameWidth = 320;
7261 const int kFrameHeight = 180;
7262
7263 // Initial rate.
Henrik Boström381d1092020-05-12 16:49:077264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077265 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7266 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7267 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 13:53:117268 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327269 /*rtt_ms=*/0,
7270 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117271
7272 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237273 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 13:53:117274 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7275 frame.set_rotation(kVideoRotation_270);
7276 video_source_.IncomingCapturedFrame(frame);
7277 WaitForEncodedFrame(timestamp_ms);
7278
7279 // Set a target rate below the minimum allowed by the codec settings.
7280 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 17:46:077281 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7282 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 16:49:077283 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 13:53:117284 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 11:36:557285 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 13:53:117286 /*link_allocation=*/target_rate,
7287 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327288 /*rtt_ms=*/0,
7289 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:117290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7291
7292 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7293 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7294 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 17:46:077295 DataRate allocation_sum =
7296 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 13:53:117297 EXPECT_EQ(min_rate, allocation_sum);
7298 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7299
7300 video_stream_encoder_->Stop();
7301}
7302
Rasmus Brandt5cad55b2019-12-19 08:47:117303TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 16:49:077304 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077305 DataRate::BitsPerSec(kTargetBitrateBps),
7306 DataRate::BitsPerSec(kTargetBitrateBps),
7307 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 10:50:237308 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 12:31:237309 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 10:50:237310 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7311 WaitForEncodedFrame(1);
7312
7313 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7314 ASSERT_TRUE(prev_rate_settings.has_value());
7315 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7316 kDefaultFramerate);
7317
7318 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7319 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7320 timestamp_ms += 1000 / kDefaultFramerate;
7321 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7322 WaitForEncodedFrame(timestamp_ms);
7323 }
7324 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7325 kDefaultFramerate);
7326 // Capture larger frame to trigger a reconfigure.
7327 codec_height_ *= 2;
7328 codec_width_ *= 2;
7329 timestamp_ms += 1000 / kDefaultFramerate;
7330 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7331 WaitForEncodedFrame(timestamp_ms);
7332
7333 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7334 auto current_rate_settings =
7335 fake_encoder_.GetAndResetLastRateControlSettings();
7336 // Ensure we have actually reconfigured twice
7337 // The rate settings should have been set again even though
7338 // they haven't changed.
7339 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 07:55:037340 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 10:50:237341
7342 video_stream_encoder_->Stop();
7343}
7344
philipeld9cc8c02019-09-16 12:53:407345struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 17:17:517346 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7347 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7348 MOCK_METHOD(void,
7349 RequestEncoderSwitch,
7350 (const webrtc::SdpVideoFormat& format),
7351 (override));
philipeld9cc8c02019-09-16 12:53:407352};
7353
philipel9b058032020-02-10 10:30:007354TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7355 constexpr int kDontCare = 100;
7356 StrictMock<MockEncoderSelector> encoder_selector;
7357 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7358 &fake_encoder_, &encoder_selector);
7359 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7360
7361 // Reset encoder for new configuration to take effect.
7362 ConfigureEncoder(video_encoder_config_.Copy());
7363
7364 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7365
7366 video_source_.IncomingCapturedFrame(
7367 CreateFrame(kDontCare, kDontCare, kDontCare));
7368 video_stream_encoder_->Stop();
7369
7370 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7371 // to it's factory, so in order for the encoder instance in the
7372 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7373 // reset the |video_stream_encoder_| here.
7374 video_stream_encoder_.reset();
7375}
7376
7377TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7378 constexpr int kDontCare = 100;
7379
7380 NiceMock<MockEncoderSelector> encoder_selector;
7381 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7382 video_send_config_.encoder_settings.encoder_switch_request_callback =
7383 &switch_callback;
7384 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7385 &fake_encoder_, &encoder_selector);
7386 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7387
7388 // Reset encoder for new configuration to take effect.
7389 ConfigureEncoder(video_encoder_config_.Copy());
7390
Mirta Dvornicic4f34d782020-02-26 12:01:197391 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 10:30:007392 .WillByDefault(Return(SdpVideoFormat("AV1")));
7393 EXPECT_CALL(switch_callback,
7394 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7395 Field(&SdpVideoFormat::name, "AV1"))));
7396
Henrik Boström381d1092020-05-12 16:49:077397 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077398 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7399 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7400 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 10:30:007401 /*fraction_lost=*/0,
7402 /*rtt_ms=*/0,
7403 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 13:56:427404 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 10:30:007405
7406 video_stream_encoder_->Stop();
7407}
7408
7409TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7410 constexpr int kSufficientBitrateToNotDrop = 1000;
7411 constexpr int kDontCare = 100;
7412
7413 NiceMock<MockVideoEncoder> video_encoder;
7414 NiceMock<MockEncoderSelector> encoder_selector;
7415 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7416 video_send_config_.encoder_settings.encoder_switch_request_callback =
7417 &switch_callback;
7418 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7419 &video_encoder, &encoder_selector);
7420 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7421
7422 // Reset encoder for new configuration to take effect.
7423 ConfigureEncoder(video_encoder_config_.Copy());
7424
7425 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7426 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7427 // not fail.
Henrik Boström381d1092020-05-12 16:49:077428 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077429 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7430 /*stable_target_bitrate=*/
7431 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7432 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 10:30:007433 /*fraction_lost=*/0,
7434 /*rtt_ms=*/0,
7435 /*cwnd_reduce_ratio=*/0);
7436
7437 ON_CALL(video_encoder, Encode(_, _))
7438 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7439 ON_CALL(encoder_selector, OnEncoderBroken())
7440 .WillByDefault(Return(SdpVideoFormat("AV2")));
7441
7442 rtc::Event encode_attempted;
7443 EXPECT_CALL(switch_callback,
7444 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7445 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7446 EXPECT_EQ(format.name, "AV2");
7447 encode_attempted.Set();
7448 });
7449
7450 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7451 encode_attempted.Wait(3000);
7452
Tomas Gunnarssond41c2a62020-09-21 13:56:427453 AdvanceTime(TimeDelta::Millis(0));
7454
philipel9b058032020-02-10 10:30:007455 video_stream_encoder_->Stop();
7456
7457 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7458 // to it's factory, so in order for the encoder instance in the
7459 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7460 // reset the |video_stream_encoder_| here.
7461 video_stream_encoder_.reset();
7462}
7463
Evan Shrubsole7c079f62019-09-26 07:55:037464TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117465 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037466 const int kFrameWidth = 320;
7467 const int kFrameHeight = 180;
7468
7469 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077470 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037472 /*target_bitrate=*/rate,
7473 /*stable_target_bitrate=*/rate,
7474 /*link_allocation=*/rate,
7475 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327476 /*rtt_ms=*/0,
7477 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037478
7479 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237480 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037481 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7482 frame.set_rotation(kVideoRotation_270);
7483 video_source_.IncomingCapturedFrame(frame);
7484 WaitForEncodedFrame(timestamp_ms);
7485 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7486
7487 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 17:46:077488 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037490 /*target_bitrate=*/new_stable_rate,
7491 /*stable_target_bitrate=*/new_stable_rate,
7492 /*link_allocation=*/rate,
7493 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327494 /*rtt_ms=*/0,
7495 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037496 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7497 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7498 video_stream_encoder_->Stop();
7499}
7500
7501TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:117502 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 07:55:037503 const int kFrameWidth = 320;
7504 const int kFrameHeight = 180;
7505
7506 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077507 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 16:49:077508 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037509 /*target_bitrate=*/rate,
7510 /*stable_target_bitrate=*/rate,
7511 /*link_allocation=*/rate,
7512 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327513 /*rtt_ms=*/0,
7514 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037515
7516 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 12:31:237517 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 07:55:037518 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7519 frame.set_rotation(kVideoRotation_270);
7520 video_source_.IncomingCapturedFrame(frame);
7521 WaitForEncodedFrame(timestamp_ms);
7522 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7523
7524 // Set a higher target rate without changing the link_allocation. Should not
7525 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:077526 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 16:49:077527 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 07:55:037528 /*target_bitrate=*/rate,
7529 /*stable_target_bitrate=*/new_stable_rate,
7530 /*link_allocation=*/rate,
7531 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:327532 /*rtt_ms=*/0,
7533 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:037534 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7535 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7536 video_stream_encoder_->Stop();
7537}
7538
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177539TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7540 test::ScopedFieldTrials field_trials(
7541 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7542 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7543 const int kFramerateFps = 30;
7544 const int kWidth = 1920;
7545 const int kHeight = 1080;
7546 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7547 // Works on screenshare mode.
7548 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7549 // We rely on the automatic resolution adaptation, but we handle framerate
7550 // adaptation manually by mocking the stats proxy.
7551 video_source_.set_adaptation_enabled(true);
7552
7553 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 16:49:077554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 17:46:077555 DataRate::BitsPerSec(kTargetBitrateBps),
7556 DataRate::BitsPerSec(kTargetBitrateBps),
7557 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177558 video_stream_encoder_->SetSource(&video_source_,
7559 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 12:08:397560 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177561
7562 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7563 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7564
7565 // Pass enough frames with the full update to trigger animation detection.
7566 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237567 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177568 frame.set_ntp_time_ms(timestamp_ms);
7569 frame.set_timestamp_us(timestamp_ms * 1000);
7570 video_source_.IncomingCapturedFrame(frame);
7571 WaitForEncodedFrame(timestamp_ms);
7572 }
7573
7574 // Resolution should be limited.
7575 rtc::VideoSinkWants expected;
7576 expected.max_framerate_fps = kFramerateFps;
7577 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 14:19:547578 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177579
7580 // Pass one frame with no known update.
7581 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 12:31:237582 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177583 frame.set_ntp_time_ms(timestamp_ms);
7584 frame.set_timestamp_us(timestamp_ms * 1000);
7585 frame.clear_update_rect();
7586
7587 video_source_.IncomingCapturedFrame(frame);
7588 WaitForEncodedFrame(timestamp_ms);
7589
7590 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 12:08:397591 EXPECT_THAT(video_source_.sink_wants(),
7592 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 15:54:177593
7594 video_stream_encoder_->Stop();
7595}
7596
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327597TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7598 const int kWidth = 720; // 540p adapted down.
7599 const int kHeight = 405;
7600 const int kNumFrames = 3;
7601 // Works on screenshare mode.
7602 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7603 /*num_spatial_layers=*/2, /*screenshare=*/true);
7604
7605 video_source_.set_adaptation_enabled(true);
7606
7607 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7608 DataRate::BitsPerSec(kTargetBitrateBps),
7609 DataRate::BitsPerSec(kTargetBitrateBps),
7610 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7611
7612 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7613
7614 // Pass enough frames with the full update to trigger animation detection.
7615 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 12:31:237616 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 10:36:327617 frame.set_ntp_time_ms(timestamp_ms);
7618 frame.set_timestamp_us(timestamp_ms * 1000);
7619 video_source_.IncomingCapturedFrame(frame);
7620 WaitForEncodedFrame(timestamp_ms);
7621 }
7622
7623 video_stream_encoder_->Stop();
7624}
7625
Yun Zhang1e4d4fd2020-09-30 07:22:467626TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7627 const float downscale_factors[] = {4.0, 2.0, 1.0};
7628 const int number_layers =
7629 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7630 VideoEncoderConfig config;
7631 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7632 for (int i = 0; i < number_layers; ++i) {
7633 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7634 config.simulcast_layers[i].active = true;
7635 }
7636 config.video_stream_factory =
7637 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7638 "VP8", /*max qp*/ 56, /*screencast*/ false,
7639 /*screenshare enabled*/ false);
7640 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7641 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7642 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7643 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7644
7645 // First initialization.
7646 // Encoder should be initialized. Next frame should be key frame.
7647 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7648 sink_.SetNumExpectedLayers(number_layers);
7649 int64_t timestamp_ms = kFrameIntervalMs;
7650 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7651 WaitForEncodedFrame(timestamp_ms);
7652 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7653 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7654 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7655 VideoFrameType::kVideoFrameKey,
7656 VideoFrameType::kVideoFrameKey}));
7657
7658 // Disable top layer.
7659 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7660 config.simulcast_layers[number_layers - 1].active = false;
7661 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7662 sink_.SetNumExpectedLayers(number_layers - 1);
7663 timestamp_ms += kFrameIntervalMs;
7664 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7665 WaitForEncodedFrame(timestamp_ms);
7666 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7667 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7668 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7669 VideoFrameType::kVideoFrameDelta,
7670 VideoFrameType::kVideoFrameDelta}));
7671
7672 // Re-enable top layer.
7673 // Encoder should be re-initialized. Next frame should be key frame.
7674 config.simulcast_layers[number_layers - 1].active = true;
7675 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7676 sink_.SetNumExpectedLayers(number_layers);
7677 timestamp_ms += kFrameIntervalMs;
7678 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7679 WaitForEncodedFrame(timestamp_ms);
7680 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7681 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7682 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7683 VideoFrameType::kVideoFrameKey,
7684 VideoFrameType::kVideoFrameKey}));
7685
7686 // Top layer max rate change.
7687 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7688 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7689 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7690 sink_.SetNumExpectedLayers(number_layers);
7691 timestamp_ms += kFrameIntervalMs;
7692 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7693 WaitForEncodedFrame(timestamp_ms);
7694 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7695 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7696 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7697 VideoFrameType::kVideoFrameDelta,
7698 VideoFrameType::kVideoFrameDelta}));
7699
7700 // Top layer resolution change.
7701 // Encoder should be re-initialized. Next frame should be key frame.
7702 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7703 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7704 sink_.SetNumExpectedLayers(number_layers);
7705 timestamp_ms += kFrameIntervalMs;
7706 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7707 WaitForEncodedFrame(timestamp_ms);
7708 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7709 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7710 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7711 VideoFrameType::kVideoFrameKey,
7712 VideoFrameType::kVideoFrameKey}));
7713 video_stream_encoder_->Stop();
7714}
7715
Henrik Boström1124ed12021-02-25 09:30:397716TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7717 const int kFrameWidth = 1280;
7718 const int kFrameHeight = 720;
7719
7720 SetUp();
7721 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7722 DataRate::BitsPerSec(kTargetBitrateBps),
7723 DataRate::BitsPerSec(kTargetBitrateBps),
7724 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7725
7726 // Capturing a frame should reconfigure the encoder and expose the encoder
7727 // resolution, which is the same as the input frame.
7728 int64_t timestamp_ms = kFrameIntervalMs;
7729 video_source_.IncomingCapturedFrame(
7730 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7731 WaitForEncodedFrame(timestamp_ms);
7732 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7733 EXPECT_THAT(video_source_.sink_wants().resolutions,
7734 ::testing::ElementsAreArray(
7735 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7736
7737 video_stream_encoder_->Stop();
7738}
7739
7740TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7741 // Pick downscale factors such that we never encode at full resolution - this
7742 // is an interesting use case. The frame resolution influences the encoder
7743 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
7744 // encoder should not ask for the frame resolution. This allows video frames
7745 // to have the appearence of one resolution but optimize its internal buffers
7746 // for what is actually encoded.
7747 const size_t kNumSimulcastLayers = 3u;
7748 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7749 const int kFrameWidth = 1280;
7750 const int kFrameHeight = 720;
7751 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7752 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7753 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7754 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7755 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7756 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7757
7758 VideoEncoderConfig config;
7759 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7760 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7761 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7762 config.simulcast_layers[i].active = true;
7763 }
7764 config.video_stream_factory =
7765 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7766 "VP8", /*max qp*/ 56, /*screencast*/ false,
7767 /*screenshare enabled*/ false);
7768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7769 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7770 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7771 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7772
7773 // Capture a frame with all layers active.
7774 int64_t timestamp_ms = kFrameIntervalMs;
7775 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7776 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7777 video_source_.IncomingCapturedFrame(
7778 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7779 WaitForEncodedFrame(timestamp_ms);
7780 // Expect encoded resolutions to match the expected simulcast layers.
7781 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7782 EXPECT_THAT(
7783 video_source_.sink_wants().resolutions,
7784 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7785
7786 // Capture a frame with one of the layers inactive.
7787 timestamp_ms += kFrameIntervalMs;
7788 config.simulcast_layers[2].active = false;
7789 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7790 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7791 video_source_.IncomingCapturedFrame(
7792 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7793 WaitForEncodedFrame(timestamp_ms);
7794
7795 // Expect encoded resolutions to match the expected simulcast layers.
7796 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7797 EXPECT_THAT(video_source_.sink_wants().resolutions,
7798 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7799
7800 // Capture a frame with all but one layer turned off.
7801 timestamp_ms += kFrameIntervalMs;
7802 config.simulcast_layers[1].active = false;
7803 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7804 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7805 video_source_.IncomingCapturedFrame(
7806 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7807 WaitForEncodedFrame(timestamp_ms);
7808
7809 // Expect encoded resolutions to match the expected simulcast layers.
7810 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7811 EXPECT_THAT(video_source_.sink_wants().resolutions,
7812 ::testing::ElementsAreArray({kLayer0Size}));
7813
7814 video_stream_encoder_->Stop();
7815}
7816
Sergey Silkin0e42cf72021-03-15 09:12:577817TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
7818 // Enable encoder source to force encoder reconfig.
7819 encoder_factory_.SetHasInternalSource(true);
7820 ResetEncoder("VP8", 1, 1, 1, false);
7821
7822 // Set QP on encoded frame and pass the frame to encode complete callback.
7823 // Since QP is present QP parsing won't be triggered and the original value
7824 // should be kept.
7825 EncodedImage encoded_image;
7826 encoded_image.qp_ = 123;
7827 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7828 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7829 CodecSpecificInfo codec_info;
7830 codec_info.codecType = kVideoCodecVP8;
7831 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7832 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7833 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7834 video_stream_encoder_->Stop();
7835}
7836
7837TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
7838 // Enable encoder source to force encoder reconfig.
7839 encoder_factory_.SetHasInternalSource(true);
7840 ResetEncoder("VP8", 1, 1, 1, false);
7841
7842 // Pass an encoded frame without QP to encode complete callback. QP should be
7843 // parsed and set.
7844 EncodedImage encoded_image;
7845 encoded_image.qp_ = -1;
7846 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7847 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7848 CodecSpecificInfo codec_info;
7849 codec_info.codecType = kVideoCodecVP8;
7850 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7851 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7852 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7853 video_stream_encoder_->Stop();
7854}
7855
7856TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7857 webrtc::test::ScopedFieldTrials field_trials(
7858 "WebRTC-QpParsingKillSwitch/Enabled/");
7859
7860 // Enable encoder source to force encoder reconfig.
7861 encoder_factory_.SetHasInternalSource(true);
7862 ResetEncoder("VP8", 1, 1, 1, false);
7863
7864 EncodedImage encoded_image;
7865 encoded_image.qp_ = -1;
7866 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7867 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7868 CodecSpecificInfo codec_info;
7869 codec_info.codecType = kVideoCodecVP8;
7870 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7871 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7872 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7873 video_stream_encoder_->Stop();
7874}
7875
Sergey Silkind19e3b92021-03-16 10:05:307876TEST_F(VideoStreamEncoderTest,
7877 QualityScalingNotAllowed_QualityScalingDisabled) {
7878 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7879
7880 // Disable scaling settings in encoder info.
7881 fake_encoder_.SetQualityScaling(false);
7882 // Disable quality scaling in encoder config.
7883 video_encoder_config.is_quality_scaling_allowed = false;
7884 ConfigureEncoder(std::move(video_encoder_config));
7885
7886 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7887 DataRate::BitsPerSec(kTargetBitrateBps),
7888 DataRate::BitsPerSec(kTargetBitrateBps),
7889 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7890
7891 test::FrameForwarder source;
7892 video_stream_encoder_->SetSource(
7893 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7894 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7895 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7896
7897 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7898 WaitForEncodedFrame(1);
7899 video_stream_encoder_->TriggerQualityLow();
7900 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7901
7902 video_stream_encoder_->Stop();
7903}
7904
7905#if !defined(WEBRTC_IOS)
7906// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
7907// disabled by default on iOS.
7908TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
7909 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7910
7911 // Disable scaling settings in encoder info.
7912 fake_encoder_.SetQualityScaling(false);
7913 // Enable quality scaling in encoder config.
7914 video_encoder_config.is_quality_scaling_allowed = true;
7915 ConfigureEncoder(std::move(video_encoder_config));
7916
7917 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7918 DataRate::BitsPerSec(kTargetBitrateBps),
7919 DataRate::BitsPerSec(kTargetBitrateBps),
7920 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7921
7922 test::FrameForwarder source;
7923 video_stream_encoder_->SetSource(
7924 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7925 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7926 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7927
7928 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7929 WaitForEncodedFrame(1);
7930 video_stream_encoder_->TriggerQualityLow();
7931 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
7932
7933 video_stream_encoder_->Stop();
7934}
7935#endif
7936
perkj26091b12016-09-01 08:17:407937} // namespace webrtc