blob: d3baa373902c7121f098f6c0ba30ca622b3e4a31 [file] [log] [blame]
perkj26091b12016-09-01 08:17:401/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 08:30:3111#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 15:24:5913#include <algorithm>
perkj803d97f2016-11-01 18:45:4614#include <limits>
Danil Chapovalovd3ba2362019-04-10 15:01:2315#include <memory>
Per512ecb32016-09-23 13:52:0616#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 14:30:1618#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 15:01:2319#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 09:20:0920#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 10:30:0021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 18:02:5622#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3123#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 16:10:5724#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 12:57:5725#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 08:30:3126#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 10:56:2027#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 15:26:0128#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 14:30:1629#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 13:59:1230#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 17:11:0031#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 17:32:3732#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Åsa Perssonc29cb2c2019-03-25 11:06:5933#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 17:11:0034#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3135#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 17:11:0036#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 15:20:1737#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 06:51:1038#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3139#include "system_wrappers/include/sleep.h"
40#include "test/encoder_settings.h"
41#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 08:51:4042#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 14:59:0043#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3144#include "test/gmock.h"
45#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 07:07:2446#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3147#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 08:17:4048
49namespace webrtc {
50
sprangb1ca0732017-02-01 16:38:1251using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 14:24:0252using ::testing::_;
philipeld9cc8c02019-09-16 12:53:4053using ::testing::AllOf;
54using ::testing::Field;
philipel9b058032020-02-10 10:30:0055using ::testing::Matcher;
56using ::testing::NiceMock;
57using ::testing::Return;
philipeld9cc8c02019-09-16 12:53:4058using ::testing::StrictMock;
kthelgason876222f2016-11-29 09:44:1159
perkj803d97f2016-11-01 18:45:4660namespace {
Åsa Persson8c1bf952018-09-13 08:42:1961const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 14:56:0062const int kQpLow = 1;
63const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 08:42:1964const int kMinFramerateFps = 2;
65const int kMinBalancedFramerateFps = 7;
66const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 12:37:0067const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 20:19:5368const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 12:18:3469const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 20:19:5370const uint32_t kSimulcastTargetBitrateBps = 3150000;
71const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 15:02:2272const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 11:21:0773const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 08:42:1974const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 08:48:4875const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 11:12:1976const VideoEncoder::ResolutionBitrateLimits
77 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
78const VideoEncoder::ResolutionBitrateLimits
79 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 12:37:0080
Mirta Dvornicic28f0eb22019-05-28 14:30:1681uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
82 0x00, 0x00, 0x03, 0x03, 0xF4,
83 0x05, 0x03, 0xC7, 0xE0, 0x1B,
84 0x41, 0x10, 0x8D, 0x00};
85
perkj803d97f2016-11-01 18:45:4686class TestBuffer : public webrtc::I420Buffer {
87 public:
88 TestBuffer(rtc::Event* event, int width, int height)
89 : I420Buffer(width, height), event_(event) {}
90
91 private:
92 friend class rtc::RefCountedObject<TestBuffer>;
93 ~TestBuffer() override {
94 if (event_)
95 event_->Set();
96 }
97 rtc::Event* const event_;
98};
99
Noah Richards51db4212019-06-12 13:59:12100// A fake native buffer that can't be converted to I420.
101class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
102 public:
103 FakeNativeBuffer(rtc::Event* event, int width, int height)
104 : event_(event), width_(width), height_(height) {}
105 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
106 int width() const override { return width_; }
107 int height() const override { return height_; }
108 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
109 return nullptr;
110 }
111
112 private:
113 friend class rtc::RefCountedObject<FakeNativeBuffer>;
114 ~FakeNativeBuffer() override {
115 if (event_)
116 event_->Set();
117 }
118 rtc::Event* const event_;
119 const int width_;
120 const int height_;
121};
122
Niels Möller7dc26b72017-12-06 09:27:48123class CpuOveruseDetectorProxy : public OveruseFrameDetector {
124 public:
Niels Möllerd1f7eb62018-03-28 14:40:58125 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
126 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 09:27:48127 last_target_framerate_fps_(-1) {}
128 virtual ~CpuOveruseDetectorProxy() {}
129
130 void OnTargetFramerateUpdated(int framerate_fps) override {
131 rtc::CritScope cs(&lock_);
132 last_target_framerate_fps_ = framerate_fps;
133 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
134 }
135
136 int GetLastTargetFramerate() {
137 rtc::CritScope cs(&lock_);
138 return last_target_framerate_fps_;
139 }
140
Niels Möller4db138e2018-04-19 07:04:13141 CpuOveruseOptions GetOptions() { return options_; }
142
Niels Möller7dc26b72017-12-06 09:27:48143 private:
144 rtc::CriticalSection lock_;
145 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
146};
147
mflodmancc3d4422017-08-03 15:27:51148class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 18:45:46149 public:
Niels Möller213618e2018-07-24 07:29:58150 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 15:01:23151 const VideoStreamEncoderSettings& settings,
152 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 17:30:41153 : VideoStreamEncoder(Clock::GetRealTimeClock(),
154 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 13:03:05155 stats_proxy,
156 settings,
Yves Gerey665174f2018-06-19 13:03:05157 std::unique_ptr<OveruseFrameDetector>(
158 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 10:50:20159 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 15:26:01160 task_queue_factory),
161 fake_cpu_resource_(
162 std::make_unique<FakeResource>(ResourceUsageState::kStable,
163 "FakeResource[CPU]")),
164 fake_quality_resource_(
165 std::make_unique<FakeResource>(ResourceUsageState::kStable,
166 "FakeResource[QP]")) {
167 InjectAdaptationResource(
168 fake_quality_resource_.get(),
169 AdaptationObserverInterface::AdaptReason::kQuality);
170 InjectAdaptationResource(fake_cpu_resource_.get(),
171 AdaptationObserverInterface::AdaptReason::kCpu);
172 }
perkj803d97f2016-11-01 18:45:46173
Henrik Boströmb08882b2020-01-07 09:11:17174 void PostTaskAndWait(bool down,
175 AdaptationObserverInterface::AdaptReason reason) {
Åsa Perssonb67c44c2019-09-24 13:25:32176 PostTaskAndWait(down, reason, /*expected_results=*/true);
177 }
178
Henrik Boströmb08882b2020-01-07 09:11:17179 void PostTaskAndWait(bool down,
180 AdaptationObserverInterface::AdaptReason reason,
181 bool expected_results) {
Niels Möllerc572ff32018-11-07 07:43:50182 rtc::Event event;
Åsa Perssonb67c44c2019-09-24 13:25:32183 encoder_queue()->PostTask([this, &event, reason, down, expected_results] {
Evan Shrubsoleaa6fbc12020-02-25 15:26:01184 ResourceUsageState usage_state =
185 down ? ResourceUsageState::kOveruse : ResourceUsageState::kUnderuse;
186
187 FakeResource* resource = nullptr;
188 switch (reason) {
189 case AdaptationObserverInterface::kQuality:
190 resource = fake_quality_resource_.get();
191 break;
192 case AdaptationObserverInterface::kCpu:
193 resource = fake_cpu_resource_.get();
194 break;
195 default:
196 RTC_NOTREACHED();
197 }
198
199 resource->set_usage_state(usage_state);
200 if (!expected_results) {
201 ASSERT_EQ(AdaptationObserverInterface::kQuality, reason)
202 << "We can only assert adaptation result for quality resources";
203 EXPECT_EQ(
204 ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency,
205 resource->last_response());
206 } else {
207 EXPECT_EQ(ResourceListenerResponse::kNothing,
208 resource->last_response());
209 }
210
perkj803d97f2016-11-01 18:45:46211 event.Set();
212 });
perkj070ba852017-02-16 23:46:27213 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 18:45:46214 }
215
kthelgason2fc52542017-03-03 08:24:41216 // This is used as a synchronisation mechanism, to make sure that the
217 // encoder queue is not blocked before we start sending it frames.
218 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 07:43:50219 rtc::Event event;
Yves Gerey665174f2018-06-19 13:03:05220 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 08:24:41221 ASSERT_TRUE(event.Wait(5000));
222 }
223
Åsa Perssonb67c44c2019-09-24 13:25:32224 void TriggerCpuOveruse() {
Henrik Boströmb08882b2020-01-07 09:11:17225 PostTaskAndWait(/*down=*/true,
226 AdaptationObserverInterface::AdaptReason::kCpu);
Åsa Perssonb67c44c2019-09-24 13:25:32227 }
kthelgason876222f2016-11-29 09:44:11228
Åsa Perssonb67c44c2019-09-24 13:25:32229 void TriggerCpuNormalUsage() {
Henrik Boströmb08882b2020-01-07 09:11:17230 PostTaskAndWait(/*down=*/false,
231 AdaptationObserverInterface::AdaptReason::kCpu);
Åsa Perssonb67c44c2019-09-24 13:25:32232 }
kthelgason876222f2016-11-29 09:44:11233
Åsa Perssonb67c44c2019-09-24 13:25:32234 void TriggerQualityLow() {
Henrik Boströmb08882b2020-01-07 09:11:17235 PostTaskAndWait(/*down=*/true,
236 AdaptationObserverInterface::AdaptReason::kQuality);
Åsa Perssonb67c44c2019-09-24 13:25:32237 }
kthelgason876222f2016-11-29 09:44:11238
Åsa Perssonb67c44c2019-09-24 13:25:32239 void TriggerQualityLowExpectFalse() {
Henrik Boströmb08882b2020-01-07 09:11:17240 PostTaskAndWait(/*down=*/true,
241 AdaptationObserverInterface::AdaptReason::kQuality,
Åsa Perssonb67c44c2019-09-24 13:25:32242 /*expected_results=*/false);
243 }
244
245 void TriggerQualityHigh() {
Henrik Boströmb08882b2020-01-07 09:11:17246 PostTaskAndWait(/*down=*/false,
247 AdaptationObserverInterface::AdaptReason::kQuality);
Åsa Perssonb67c44c2019-09-24 13:25:32248 }
sprangfda496a2017-06-15 11:21:07249
Niels Möller7dc26b72017-12-06 09:27:48250 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Evan Shrubsoleaa6fbc12020-02-25 15:26:01251 std::unique_ptr<FakeResource> fake_cpu_resource_;
252 std::unique_ptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 18:45:46253};
254
asapersson5f7226f2016-11-25 12:37:00255class VideoStreamFactory
256 : public VideoEncoderConfig::VideoStreamFactoryInterface {
257 public:
sprangfda496a2017-06-15 11:21:07258 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
259 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 12:37:00260 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 11:21:07261 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 12:37:00262 }
263
264 private:
265 std::vector<VideoStream> CreateEncoderStreams(
266 int width,
267 int height,
268 const VideoEncoderConfig& encoder_config) override {
269 std::vector<VideoStream> streams =
270 test::CreateVideoStreams(width, height, encoder_config);
271 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 14:11:29272 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 11:21:07273 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 12:37:00274 }
275 return streams;
276 }
sprangfda496a2017-06-15 11:21:07277
asapersson5f7226f2016-11-25 12:37:00278 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 11:21:07279 const int framerate_;
asapersson5f7226f2016-11-25 12:37:00280};
281
Noah Richards51db4212019-06-12 13:59:12282// Simulates simulcast behavior and makes highest stream resolutions divisible
283// by 4.
284class CroppingVideoStreamFactory
285 : public VideoEncoderConfig::VideoStreamFactoryInterface {
286 public:
287 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
288 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
289 EXPECT_GT(num_temporal_layers, 0u);
290 EXPECT_GT(framerate, 0);
291 }
292
293 private:
294 std::vector<VideoStream> CreateEncoderStreams(
295 int width,
296 int height,
297 const VideoEncoderConfig& encoder_config) override {
298 std::vector<VideoStream> streams = test::CreateVideoStreams(
299 width - width % 4, height - height % 4, encoder_config);
300 for (VideoStream& stream : streams) {
301 stream.num_temporal_layers = num_temporal_layers_;
302 stream.max_framerate = framerate_;
303 }
304 return streams;
305 }
306
307 const size_t num_temporal_layers_;
308 const int framerate_;
309};
310
sprangb1ca0732017-02-01 16:38:12311class AdaptingFrameForwarder : public test::FrameForwarder {
312 public:
313 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 12:51:49314 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 16:38:12315
316 void set_adaptation_enabled(bool enabled) {
317 rtc::CritScope cs(&crit_);
318 adaptation_enabled_ = enabled;
319 }
320
asaperssonfab67072017-04-04 12:51:49321 bool adaption_enabled() const {
sprangb1ca0732017-02-01 16:38:12322 rtc::CritScope cs(&crit_);
323 return adaptation_enabled_;
324 }
325
asapersson09f05612017-05-16 06:40:18326 rtc::VideoSinkWants last_wants() const {
327 rtc::CritScope cs(&crit_);
328 return last_wants_;
329 }
330
Danil Chapovalovb9b146c2018-06-15 10:28:07331 absl::optional<int> last_sent_width() const { return last_width_; }
332 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-09 01:04:29333
sprangb1ca0732017-02-01 16:38:12334 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
335 int cropped_width = 0;
336 int cropped_height = 0;
337 int out_width = 0;
338 int out_height = 0;
sprangc5d62e22017-04-03 06:53:04339 if (adaption_enabled()) {
340 if (adapter_.AdaptFrameResolution(
341 video_frame.width(), video_frame.height(),
342 video_frame.timestamp_us() * 1000, &cropped_width,
343 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 22:49:37344 VideoFrame adapted_frame =
345 VideoFrame::Builder()
346 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
347 nullptr, out_width, out_height))
348 .set_timestamp_rtp(99)
349 .set_timestamp_ms(99)
350 .set_rotation(kVideoRotation_0)
351 .build();
sprangc5d62e22017-04-03 06:53:04352 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 15:54:17353 if (video_frame.has_update_rect()) {
354 adapted_frame.set_update_rect(
355 video_frame.update_rect().ScaleWithFrame(
356 video_frame.width(), video_frame.height(), 0, 0,
357 video_frame.width(), video_frame.height(), out_width,
358 out_height));
359 }
sprangc5d62e22017-04-03 06:53:04360 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-09 01:04:29361 last_width_.emplace(adapted_frame.width());
362 last_height_.emplace(adapted_frame.height());
363 } else {
Danil Chapovalovb9b146c2018-06-15 10:28:07364 last_width_ = absl::nullopt;
365 last_height_ = absl::nullopt;
sprangc5d62e22017-04-03 06:53:04366 }
sprangb1ca0732017-02-01 16:38:12367 } else {
368 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-09 01:04:29369 last_width_.emplace(video_frame.width());
370 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 16:38:12371 }
372 }
373
374 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
375 const rtc::VideoSinkWants& wants) override {
376 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-16 06:40:18377 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 15:56:01378 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 16:38:12379 test::FrameForwarder::AddOrUpdateSink(sink, wants);
380 }
sprangb1ca0732017-02-01 16:38:12381 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 11:17:22382 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
383 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 10:28:07384 absl::optional<int> last_width_;
385 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 16:38:12386};
sprangc5d62e22017-04-03 06:53:04387
Niels Möller213618e2018-07-24 07:29:58388// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-03 06:53:04389class MockableSendStatisticsProxy : public SendStatisticsProxy {
390 public:
391 MockableSendStatisticsProxy(Clock* clock,
392 const VideoSendStream::Config& config,
393 VideoEncoderConfig::ContentType content_type)
394 : SendStatisticsProxy(clock, config, content_type) {}
395
396 VideoSendStream::Stats GetStats() override {
397 rtc::CritScope cs(&lock_);
398 if (mock_stats_)
399 return *mock_stats_;
400 return SendStatisticsProxy::GetStats();
401 }
402
Niels Möller213618e2018-07-24 07:29:58403 int GetInputFrameRate() const override {
404 rtc::CritScope cs(&lock_);
405 if (mock_stats_)
406 return mock_stats_->input_frame_rate;
407 return SendStatisticsProxy::GetInputFrameRate();
408 }
sprangc5d62e22017-04-03 06:53:04409 void SetMockStats(const VideoSendStream::Stats& stats) {
410 rtc::CritScope cs(&lock_);
411 mock_stats_.emplace(stats);
412 }
413
414 void ResetMockStats() {
415 rtc::CritScope cs(&lock_);
416 mock_stats_.reset();
417 }
418
419 private:
420 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 10:28:07421 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-03 06:53:04422};
423
sprang4847ae62017-06-27 14:06:52424class MockBitrateObserver : public VideoBitrateAllocationObserver {
425 public:
Erik Språng566124a2018-04-23 10:32:22426 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 14:06:52427};
428
philipel9b058032020-02-10 10:30:00429class MockEncoderSelector
430 : public VideoEncoderFactory::EncoderSelectorInterface {
431 public:
432 MOCK_METHOD1(OnCurrentEncoder, void(const SdpVideoFormat& format));
Mirta Dvornicic4f34d782020-02-26 12:01:19433 MOCK_METHOD1(OnAvailableBitrate,
philipel9b058032020-02-10 10:30:00434 absl::optional<SdpVideoFormat>(const DataRate& rate));
435 MOCK_METHOD0(OnEncoderBroken, absl::optional<SdpVideoFormat>());
436};
437
perkj803d97f2016-11-01 18:45:46438} // namespace
439
mflodmancc3d4422017-08-03 15:27:51440class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 08:17:40441 public:
442 static const int kDefaultTimeoutMs = 30 * 1000;
443
mflodmancc3d4422017-08-03 15:27:51444 VideoStreamEncoderTest()
perkj26091b12016-09-01 08:17:40445 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-03 06:45:26446 codec_width_(320),
447 codec_height_(240),
Åsa Persson8c1bf952018-09-13 08:42:19448 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 15:01:23449 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 08:17:40450 fake_encoder_(),
Niels Möller4db138e2018-04-19 07:04:13451 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-03 06:53:04452 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 18:45:46453 Clock::GetRealTimeClock(),
454 video_send_config_,
455 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 08:17:40456 sink_(&fake_encoder_) {}
457
458 void SetUp() override {
perkj803d97f2016-11-01 18:45:46459 metrics::Reset();
perkj26091b12016-09-01 08:17:40460 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 07:04:13461 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56462 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 12:18:34463 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 13:36:51464 video_send_config_.rtp.payload_name = "FAKE";
465 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 08:17:40466
Per512ecb32016-09-23 13:52:06467 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51468 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 14:06:52469 video_encoder_config.video_stream_factory =
470 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 15:41:30471 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 14:06:52472
473 // Framerate limit is specified by the VideoStreamFactory.
474 std::vector<VideoStream> streams =
475 video_encoder_config.video_stream_factory->CreateEncoderStreams(
476 codec_width_, codec_height_, video_encoder_config);
477 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 10:16:00478 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 14:06:52479
Niels Möllerf1338562018-04-26 07:51:47480 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 12:37:00481 }
482
Niels Möllerf1338562018-04-26 07:51:47483 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 15:27:51484 if (video_stream_encoder_)
485 video_stream_encoder_->Stop();
486 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 15:01:23487 stats_proxy_.get(), video_send_config_.encoder_settings,
488 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 15:27:51489 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
490 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:41491 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 15:27:51492 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
493 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:47494 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:51495 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 12:37:00496 }
497
498 void ResetEncoder(const std::string& payload_name,
499 size_t num_streams,
500 size_t num_temporal_layers,
emircanbbcc3562017-08-18 07:28:40501 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 14:06:52502 bool screenshare) {
Niels Möller259a4972018-04-05 13:36:51503 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 12:37:00504
505 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51506 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 12:37:00507 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 20:19:53508 video_encoder_config.max_bitrate_bps =
509 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 12:37:00510 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 11:21:07511 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
512 kDefaultFramerate);
sprang4847ae62017-06-27 14:06:52513 video_encoder_config.content_type =
514 screenshare ? VideoEncoderConfig::ContentType::kScreen
515 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 07:28:40516 if (payload_name == "VP9") {
517 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
518 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
519 video_encoder_config.encoder_specific_settings =
520 new rtc::RefCountedObject<
521 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
522 }
Niels Möllerf1338562018-04-26 07:51:47523 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 08:17:40524 }
525
sprang57c2fff2017-01-16 14:24:02526 VideoFrame CreateFrame(int64_t ntp_time_ms,
527 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 22:49:37528 VideoFrame frame =
529 VideoFrame::Builder()
530 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
531 destruction_event, codec_width_, codec_height_))
532 .set_timestamp_rtp(99)
533 .set_timestamp_ms(99)
534 .set_rotation(kVideoRotation_0)
535 .build();
sprang57c2fff2017-01-16 14:24:02536 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 08:17:40537 return frame;
538 }
539
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26540 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
541 rtc::Event* destruction_event,
542 int offset_x) const {
543 VideoFrame frame =
544 VideoFrame::Builder()
545 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
546 destruction_event, codec_width_, codec_height_))
547 .set_timestamp_rtp(99)
548 .set_timestamp_ms(99)
549 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 09:34:12550 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26551 .build();
552 frame.set_ntp_time_ms(ntp_time_ms);
553 return frame;
554 }
555
sprang57c2fff2017-01-16 14:24:02556 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 22:49:37557 VideoFrame frame =
558 VideoFrame::Builder()
559 .set_video_frame_buffer(
560 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
561 .set_timestamp_rtp(99)
562 .set_timestamp_ms(99)
563 .set_rotation(kVideoRotation_0)
564 .build();
sprang57c2fff2017-01-16 14:24:02565 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-03 06:53:04566 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 18:45:46567 return frame;
568 }
569
Noah Richards51db4212019-06-12 13:59:12570 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
571 rtc::Event* destruction_event,
572 int width,
573 int height) const {
574 VideoFrame frame =
575 VideoFrame::Builder()
576 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
577 destruction_event, width, height))
578 .set_timestamp_rtp(99)
579 .set_timestamp_ms(99)
580 .set_rotation(kVideoRotation_0)
581 .build();
582 frame.set_ntp_time_ms(ntp_time_ms);
583 return frame;
584 }
585
586 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
587 rtc::Event* destruction_event) const {
588 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
589 codec_height_);
590 }
591
Åsa Perssonc29cb2c2019-03-25 11:06:59592 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
593 MockBitrateObserver bitrate_observer;
594 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
595
596 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
597 .Times(1);
Florent Castellia8336d32019-09-09 11:36:55598 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:07599 DataRate::BitsPerSec(kTargetBitrateBps),
600 DataRate::BitsPerSec(kTargetBitrateBps),
601 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 11:06:59602
603 video_source_.IncomingCapturedFrame(
604 CreateFrame(1, codec_width_, codec_height_));
605 WaitForEncodedFrame(1);
606 }
607
asapersson02465b82017-04-10 08:12:52608 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 08:12:52609 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-16 06:40:18610 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
611 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 08:12:52612 }
613
asapersson09f05612017-05-16 06:40:18614 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
615 const rtc::VideoSinkWants& wants2) {
616 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
617 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
618 }
619
Åsa Persson8c1bf952018-09-13 08:42:19620 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
621 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
622 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
623 EXPECT_FALSE(wants.target_pixel_count);
624 }
625
asapersson09f05612017-05-16 06:40:18626 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
627 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 08:42:19628 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-16 06:40:18629 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
630 EXPECT_GT(wants1.max_pixel_count, 0);
631 }
632
633 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
634 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 08:42:19635 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-16 06:40:18636 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
637 }
638
asaperssonf7e294d2017-06-14 06:25:22639 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
640 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 08:42:19641 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:22642 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
643 }
644
645 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
646 const rtc::VideoSinkWants& wants2) {
647 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
648 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
649 }
650
651 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
652 const rtc::VideoSinkWants& wants2) {
653 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
654 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
655 }
656
657 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
658 const rtc::VideoSinkWants& wants2) {
659 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
660 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
661 EXPECT_GT(wants1.max_pixel_count, 0);
662 }
663
664 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
665 const rtc::VideoSinkWants& wants2) {
666 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
667 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
668 }
669
asapersson09f05612017-05-16 06:40:18670 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
671 int pixel_count) {
Åsa Persson8c1bf952018-09-13 08:42:19672 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 08:12:52673 EXPECT_LT(wants.max_pixel_count, pixel_count);
674 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-16 06:40:18675 }
676
677 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
678 EXPECT_LT(wants.max_framerate_fps, fps);
679 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
680 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 08:12:52681 }
682
asaperssonf7e294d2017-06-14 06:25:22683 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
684 int expected_fps) {
685 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
686 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
687 EXPECT_FALSE(wants.target_pixel_count);
688 }
689
Jonathan Yubc771b72017-12-09 01:04:29690 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
691 int last_frame_pixels) {
692 // Balanced mode should always scale FPS to the desired range before
693 // attempting to scale resolution.
694 int fps_limit = wants.max_framerate_fps;
695 if (last_frame_pixels <= 320 * 240) {
Henrik Boström60383832020-02-28 08:03:53696 EXPECT_LE(7, fps_limit);
697 EXPECT_LE(fps_limit, 10);
Jonathan Yubc771b72017-12-09 01:04:29698 } else if (last_frame_pixels <= 480 * 270) {
Henrik Boström60383832020-02-28 08:03:53699 EXPECT_LE(10, fps_limit);
700 EXPECT_LE(fps_limit, 15);
Jonathan Yubc771b72017-12-09 01:04:29701 } else if (last_frame_pixels <= 640 * 480) {
702 EXPECT_LE(15, fps_limit);
703 } else {
Åsa Persson8c1bf952018-09-13 08:42:19704 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-09 01:04:29705 }
706 }
707
sprang4847ae62017-06-27 14:06:52708 void WaitForEncodedFrame(int64_t expected_ntp_time) {
709 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 10:16:00710 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52711 }
712
713 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
714 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 10:16:00715 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52716 return ok;
717 }
718
719 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
720 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 10:16:00721 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52722 }
723
724 void ExpectDroppedFrame() {
725 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 10:16:00726 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52727 }
728
729 bool WaitForFrame(int64_t timeout_ms) {
730 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 10:16:00731 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52732 return ok;
733 }
734
perkj26091b12016-09-01 08:17:40735 class TestEncoder : public test::FakeEncoder {
736 public:
Niels Möllerc572ff32018-11-07 07:43:50737 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 08:17:40738
asaperssonfab67072017-04-04 12:51:49739 VideoCodec codec_config() const {
brandtre78d2662017-01-16 13:57:16740 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-03 06:45:26741 return config_;
742 }
743
744 void BlockNextEncode() {
brandtre78d2662017-01-16 13:57:16745 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-03 06:45:26746 block_next_encode_ = true;
747 }
748
Erik Språngaed30702018-11-05 11:57:17749 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 08:24:41750 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 11:42:18751 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 14:52:33752 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 11:42:18753 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 14:56:00754 info.scaling_settings = VideoEncoder::ScalingSettings(
755 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 11:42:18756 }
757 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 11:06:59758 for (int i = 0; i < kMaxSpatialLayers; ++i) {
759 if (temporal_layers_supported_[i]) {
760 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
761 info.fps_allocation[i].resize(num_layers);
762 }
763 }
Erik Språngaed30702018-11-05 11:57:17764 }
Sergey Silkin6456e352019-07-08 15:56:40765
766 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 08:47:11767 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 11:57:17768 return info;
kthelgason876222f2016-11-29 09:44:11769 }
770
Erik Språngb7cb7b52019-02-26 14:52:33771 int32_t RegisterEncodeCompleteCallback(
772 EncodedImageCallback* callback) override {
773 rtc::CritScope lock(&local_crit_sect_);
774 encoded_image_callback_ = callback;
775 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
776 }
777
perkjfa10b552016-10-03 06:45:26778 void ContinueEncode() { continue_encode_event_.Set(); }
779
780 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
781 uint32_t timestamp) const {
brandtre78d2662017-01-16 13:57:16782 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-03 06:45:26783 EXPECT_EQ(timestamp_, timestamp);
784 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
785 }
786
kthelgason2fc52542017-03-03 08:24:41787 void SetQualityScaling(bool b) {
788 rtc::CritScope lock(&local_crit_sect_);
789 quality_scaling_ = b;
790 }
kthelgasonad9010c2017-02-14 08:46:51791
Rasmus Brandt5cad55b2019-12-19 08:47:11792 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
793 rtc::CritScope lock(&local_crit_sect_);
794 requested_resolution_alignment_ = requested_resolution_alignment;
795 }
796
Mirta Dvornicicccc1b572019-01-15 11:42:18797 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
798 rtc::CritScope lock(&local_crit_sect_);
799 is_hardware_accelerated_ = is_hardware_accelerated;
800 }
801
Åsa Perssonc29cb2c2019-03-25 11:06:59802 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
803 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
804 rtc::CritScope lock(&local_crit_sect_);
805 temporal_layers_supported_[spatial_idx] = supported;
806 }
807
Sergey Silkin6456e352019-07-08 15:56:40808 void SetResolutionBitrateLimits(
809 std::vector<ResolutionBitrateLimits> thresholds) {
810 rtc::CritScope cs(&local_crit_sect_);
811 resolution_bitrate_limits_ = thresholds;
812 }
813
sprangfe627f32017-03-29 15:24:59814 void ForceInitEncodeFailure(bool force_failure) {
815 rtc::CritScope lock(&local_crit_sect_);
816 force_init_encode_failed_ = force_failure;
817 }
818
Niels Möller6bb5ab92019-01-11 10:11:10819 void SimulateOvershoot(double rate_factor) {
820 rtc::CritScope lock(&local_crit_sect_);
821 rate_factor_ = rate_factor;
822 }
823
Erik Språngd7329ca2019-02-21 20:19:53824 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 10:11:10825 rtc::CritScope lock(&local_crit_sect_);
826 return last_framerate_;
827 }
828
Erik Språngd7329ca2019-02-21 20:19:53829 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26830 rtc::CritScope lock(&local_crit_sect_);
831 return last_update_rect_;
832 }
833
Niels Möller87e2d782019-03-07 09:18:23834 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 20:19:53835 rtc::CritScope lock(&local_crit_sect_);
836 return last_frame_types_;
837 }
838
839 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 09:18:23840 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 14:43:58841 keyframe ? VideoFrameType::kVideoFrameKey
842 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:53843 {
844 rtc::CritScope lock(&local_crit_sect_);
845 last_frame_types_ = frame_type;
846 }
Niels Möllerb859b322019-03-07 11:40:01847 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 20:19:53848 }
849
Erik Språngb7cb7b52019-02-26 14:52:33850 void InjectEncodedImage(const EncodedImage& image) {
851 rtc::CritScope lock(&local_crit_sect_);
852 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
853 }
854
Mirta Dvornicic28f0eb22019-05-28 14:30:16855 void InjectEncodedImage(const EncodedImage& image,
856 const CodecSpecificInfo* codec_specific_info,
857 const RTPFragmentationHeader* fragmentation) {
858 rtc::CritScope lock(&local_crit_sect_);
859 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
860 fragmentation);
861 }
862
Erik Språngd7329ca2019-02-21 20:19:53863 void ExpectNullFrame() {
864 rtc::CritScope lock(&local_crit_sect_);
865 expect_null_frame_ = true;
866 }
867
Erik Språng5056af02019-09-02 13:53:11868 absl::optional<VideoEncoder::RateControlParameters>
869 GetAndResetLastRateControlSettings() {
870 auto settings = last_rate_control_settings_;
871 last_rate_control_settings_.reset();
872 return settings;
Erik Språngd7329ca2019-02-21 20:19:53873 }
874
Sergey Silkin5ee69672019-07-02 12:18:34875 int GetNumEncoderInitializations() const {
876 rtc::CritScope lock(&local_crit_sect_);
877 return num_encoder_initializations_;
878 }
879
Evan Shrubsole7c079f62019-09-26 07:55:03880 int GetNumSetRates() const {
881 rtc::CritScope lock(&local_crit_sect_);
882 return num_set_rates_;
883 }
884
perkjfa10b552016-10-03 06:45:26885 private:
perkj26091b12016-09-01 08:17:40886 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 09:18:23887 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 08:17:40888 bool block_encode;
889 {
brandtre78d2662017-01-16 13:57:16890 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 20:19:53891 if (expect_null_frame_) {
892 EXPECT_EQ(input_image.timestamp(), 0u);
893 EXPECT_EQ(input_image.width(), 1);
894 last_frame_types_ = *frame_types;
895 expect_null_frame_ = false;
896 } else {
897 EXPECT_GT(input_image.timestamp(), timestamp_);
898 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
899 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
900 }
perkj26091b12016-09-01 08:17:40901
902 timestamp_ = input_image.timestamp();
903 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 18:45:46904 last_input_width_ = input_image.width();
905 last_input_height_ = input_image.height();
perkj26091b12016-09-01 08:17:40906 block_encode = block_next_encode_;
907 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26908 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 20:19:53909 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 08:17:40910 }
Niels Möllerb859b322019-03-07 11:40:01911 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 08:17:40912 if (block_encode)
perkja49cbd32016-09-16 14:53:41913 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:40914 return result;
915 }
916
sprangfe627f32017-03-29 15:24:59917 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 12:57:57918 const Settings& settings) override {
919 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 12:18:34920
sprangfe627f32017-03-29 15:24:59921 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 14:52:33922 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 12:18:34923
924 ++num_encoder_initializations_;
925
Erik Språng82fad3d2018-03-21 08:57:23926 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 15:24:59927 // Simulate setting up temporal layers, in order to validate the life
928 // cycle of these objects.
Elad Aloncde8ab22019-03-20 10:56:20929 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 09:20:09930 frame_buffer_controller_ =
931 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 15:24:59932 }
Erik Språngb7cb7b52019-02-26 14:52:33933 if (force_init_encode_failed_) {
934 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 15:24:59935 return -1;
Erik Språngb7cb7b52019-02-26 14:52:33936 }
Mirta Dvornicicccc1b572019-01-15 11:42:18937
Erik Språngb7cb7b52019-02-26 14:52:33938 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 15:24:59939 return res;
940 }
941
Erik Språngb7cb7b52019-02-26 14:52:33942 int32_t Release() override {
943 rtc::CritScope lock(&local_crit_sect_);
944 EXPECT_NE(initialized_, EncoderState::kUninitialized);
945 initialized_ = EncoderState::kUninitialized;
946 return FakeEncoder::Release();
947 }
948
Erik Språng16cb8f52019-04-12 11:59:09949 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 10:11:10950 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 07:55:03951 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 10:11:10952 VideoBitrateAllocation adjusted_rate_allocation;
953 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
954 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 11:59:09955 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 10:11:10956 adjusted_rate_allocation.SetBitrate(
957 si, ti,
Erik Språng16cb8f52019-04-12 11:59:09958 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 10:11:10959 rate_factor_));
960 }
961 }
962 }
Erik Språng16cb8f52019-04-12 11:59:09963 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 13:53:11964 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 11:59:09965 RateControlParameters adjusted_paramters = parameters;
966 adjusted_paramters.bitrate = adjusted_rate_allocation;
967 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 10:11:10968 }
969
brandtre78d2662017-01-16 13:57:16970 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 14:52:33971 enum class EncoderState {
972 kUninitialized,
973 kInitializationFailed,
974 kInitialized
975 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
976 EncoderState::kUninitialized;
danilchapa37de392017-09-09 11:17:22977 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 08:17:40978 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 11:17:22979 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
980 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
981 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
982 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
983 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 08:47:11984 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 11:42:18985 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 10:56:20986 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 11:17:22987 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 11:06:59988 absl::optional<bool>
989 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
990 local_crit_sect_);
danilchapa37de392017-09-09 11:17:22991 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 10:11:10992 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
993 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 13:53:11994 absl::optional<VideoEncoder::RateControlParameters>
995 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26996 VideoFrame::UpdateRect last_update_rect_
997 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 09:18:23998 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 20:19:53999 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 14:52:331000 EncodedImageCallback* encoded_image_callback_
1001 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 15:18:361002 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 12:18:341003 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 15:56:401004 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1005 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 07:55:031006 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 08:17:401007 };
1008
mflodmancc3d4422017-08-03 15:27:511009 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 08:17:401010 public:
1011 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 07:43:501012 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 08:17:401013
perkj26091b12016-09-01 08:17:401014 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 14:06:521015 EXPECT_TRUE(
1016 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1017 }
1018
1019 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1020 int64_t timeout_ms) {
perkj26091b12016-09-01 08:17:401021 uint32_t timestamp = 0;
sprang4847ae62017-06-27 14:06:521022 if (!encoded_frame_event_.Wait(timeout_ms))
1023 return false;
perkj26091b12016-09-01 08:17:401024 {
1025 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 16:38:121026 timestamp = last_timestamp_;
perkj26091b12016-09-01 08:17:401027 }
1028 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 14:06:521029 return true;
perkj26091b12016-09-01 08:17:401030 }
1031
sprangb1ca0732017-02-01 16:38:121032 void WaitForEncodedFrame(uint32_t expected_width,
1033 uint32_t expected_height) {
sprangc5d62e22017-04-03 06:53:041034 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 13:13:561035 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-03 06:53:041036 }
1037
Åsa Perssonc74d8da2017-12-04 13:13:561038 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-03 06:53:041039 uint32_t expected_height) {
sprangb1ca0732017-02-01 16:38:121040 uint32_t width = 0;
1041 uint32_t height = 0;
sprangb1ca0732017-02-01 16:38:121042 {
1043 rtc::CritScope lock(&crit_);
1044 width = last_width_;
1045 height = last_height_;
1046 }
1047 EXPECT_EQ(expected_height, height);
1048 EXPECT_EQ(expected_width, width);
1049 }
1050
Rasmus Brandt5cad55b2019-12-19 08:47:111051 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1052 int width = 0;
1053 int height = 0;
1054 {
1055 rtc::CritScope lock(&crit_);
1056 width = last_width_;
1057 height = last_height_;
1058 }
1059 EXPECT_EQ(width % resolution_alignment, 0);
1060 EXPECT_EQ(height % resolution_alignment, 0);
1061 }
1062
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141063 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1064 VideoRotation rotation;
1065 {
1066 rtc::CritScope lock(&crit_);
1067 rotation = last_rotation_;
1068 }
1069 EXPECT_EQ(expected_rotation, rotation);
1070 }
1071
kthelgason2fc52542017-03-03 08:24:411072 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 15:02:221073
sprangc5d62e22017-04-03 06:53:041074 bool WaitForFrame(int64_t timeout_ms) {
1075 return encoded_frame_event_.Wait(timeout_ms);
1076 }
1077
perkj26091b12016-09-01 08:17:401078 void SetExpectNoFrames() {
1079 rtc::CritScope lock(&crit_);
1080 expect_frames_ = false;
1081 }
1082
asaperssonfab67072017-04-04 12:51:491083 int number_of_reconfigurations() const {
Per512ecb32016-09-23 13:52:061084 rtc::CritScope lock(&crit_);
1085 return number_of_reconfigurations_;
1086 }
1087
asaperssonfab67072017-04-04 12:51:491088 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 13:52:061089 rtc::CritScope lock(&crit_);
1090 return min_transmit_bitrate_bps_;
1091 }
1092
Erik Språngd7329ca2019-02-21 20:19:531093 void SetNumExpectedLayers(size_t num_layers) {
1094 rtc::CritScope lock(&crit_);
1095 num_expected_layers_ = num_layers;
1096 }
1097
Erik Språngb7cb7b52019-02-26 14:52:331098 int64_t GetLastCaptureTimeMs() const {
1099 rtc::CritScope lock(&crit_);
1100 return last_capture_time_ms_;
1101 }
1102
Mirta Dvornicic28f0eb22019-05-28 14:30:161103 std::vector<uint8_t> GetLastEncodedImageData() {
1104 rtc::CritScope lock(&crit_);
1105 return std::move(last_encoded_image_data_);
1106 }
1107
1108 RTPFragmentationHeader GetLastFragmentation() {
1109 rtc::CritScope lock(&crit_);
1110 return std::move(last_fragmentation_);
1111 }
1112
perkj26091b12016-09-01 08:17:401113 private:
sergeyu2cb155a2016-11-04 18:39:291114 Result OnEncodedImage(
1115 const EncodedImage& encoded_image,
1116 const CodecSpecificInfo* codec_specific_info,
1117 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 13:52:061118 rtc::CritScope lock(&crit_);
1119 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 14:30:161120 last_encoded_image_data_ = std::vector<uint8_t>(
1121 encoded_image.data(), encoded_image.data() + encoded_image.size());
1122 if (fragmentation) {
1123 last_fragmentation_.CopyFrom(*fragmentation);
1124 }
Erik Språngd7329ca2019-02-21 20:19:531125 uint32_t timestamp = encoded_image.Timestamp();
1126 if (last_timestamp_ != timestamp) {
1127 num_received_layers_ = 1;
1128 } else {
1129 ++num_received_layers_;
1130 }
1131 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 14:52:331132 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 16:38:121133 last_width_ = encoded_image._encodedWidth;
1134 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141135 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 20:19:531136 if (num_received_layers_ == num_expected_layers_) {
1137 encoded_frame_event_.Set();
1138 }
sprangb1ca0732017-02-01 16:38:121139 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 13:52:061140 }
1141
Rasmus Brandtc402dbe2019-02-04 10:09:461142 void OnEncoderConfigurationChanged(
1143 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 12:10:271144 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 10:09:461145 VideoEncoderConfig::ContentType content_type,
1146 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 12:18:341147 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 13:52:061148 ++number_of_reconfigurations_;
1149 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1150 }
1151
perkj26091b12016-09-01 08:17:401152 rtc::CriticalSection crit_;
1153 TestEncoder* test_encoder_;
1154 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 14:30:161155 std::vector<uint8_t> last_encoded_image_data_;
1156 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 16:38:121157 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 14:52:331158 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 16:38:121159 uint32_t last_height_ = 0;
1160 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141161 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 20:19:531162 size_t num_expected_layers_ = 1;
1163 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 08:17:401164 bool expect_frames_ = true;
Per512ecb32016-09-23 13:52:061165 int number_of_reconfigurations_ = 0;
1166 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 08:17:401167 };
1168
Sergey Silkin5ee69672019-07-02 12:18:341169 class VideoBitrateAllocatorProxyFactory
1170 : public VideoBitrateAllocatorFactory {
1171 public:
1172 VideoBitrateAllocatorProxyFactory()
1173 : bitrate_allocator_factory_(
1174 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1175
1176 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1177 const VideoCodec& codec) override {
1178 rtc::CritScope lock(&crit_);
1179 codec_config_ = codec;
1180 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1181 }
1182
1183 VideoCodec codec_config() const {
1184 rtc::CritScope lock(&crit_);
1185 return codec_config_;
1186 }
1187
1188 private:
1189 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1190
1191 rtc::CriticalSection crit_;
1192 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1193 };
1194
perkj26091b12016-09-01 08:17:401195 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 15:41:301196 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 13:52:061197 int codec_width_;
1198 int codec_height_;
sprang4847ae62017-06-27 14:06:521199 int max_framerate_;
Erik Språng82268752019-08-29 13:07:471200 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 15:01:231201 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 08:17:401202 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 07:07:241203 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 12:18:341204 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-03 06:53:041205 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 08:17:401206 TestSink sink_;
sprangb1ca0732017-02-01 16:38:121207 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 15:27:511208 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 08:17:401209};
1210
mflodmancc3d4422017-08-03 15:27:511211TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 13:14:011212 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071213 DataRate::BitsPerSec(kTargetBitrateBps),
1214 DataRate::BitsPerSec(kTargetBitrateBps),
1215 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 07:43:501216 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411217 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 14:06:521218 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 14:53:411219 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 15:27:511220 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401221}
1222
mflodmancc3d4422017-08-03 15:27:511223TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 08:17:401224 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 07:43:501225 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 11:05:491226 // The encoder will cache up to one frame for a short duration. Adding two
1227 // frames means that the first frame will be dropped and the second frame will
1228 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 14:53:411229 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 11:05:491230 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 14:53:411231 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401232
Erik Språng4c6ca302019-04-08 13:14:011233 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071234 DataRate::BitsPerSec(kTargetBitrateBps),
1235 DataRate::BitsPerSec(kTargetBitrateBps),
1236 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401237
Sebastian Janssona3177052018-04-10 11:05:491238 // The pending frame should be received.
sprang4847ae62017-06-27 14:06:521239 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 11:05:491240 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1241
1242 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:511243 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401244}
1245
mflodmancc3d4422017-08-03 15:27:511246TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 13:14:011247 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071248 DataRate::BitsPerSec(kTargetBitrateBps),
1249 DataRate::BitsPerSec(kTargetBitrateBps),
1250 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 14:53:411251 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521252 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401253
Danil Chapovalovcad3e0e2020-02-17 17:46:071254 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
1255 DataRate::BitsPerSec(0),
1256 DataRate::BitsPerSec(0), 0, 0, 0);
Sebastian Janssona3177052018-04-10 11:05:491257 // The encoder will cache up to one frame for a short duration. Adding two
1258 // frames means that the first frame will be dropped and the second frame will
1259 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 14:53:411260 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 11:05:491261 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401262
Erik Språng4c6ca302019-04-08 13:14:011263 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071264 DataRate::BitsPerSec(kTargetBitrateBps),
1265 DataRate::BitsPerSec(kTargetBitrateBps),
1266 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:521267 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 11:05:491268 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1269 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 15:27:511270 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401271}
1272
mflodmancc3d4422017-08-03 15:27:511273TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 13:14:011274 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071275 DataRate::BitsPerSec(kTargetBitrateBps),
1276 DataRate::BitsPerSec(kTargetBitrateBps),
1277 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 14:53:411278 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521279 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401280
1281 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 14:53:411282 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 08:17:401283
perkja49cbd32016-09-16 14:53:411284 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521285 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511286 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401287}
1288
mflodmancc3d4422017-08-03 15:27:511289TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 13:14:011290 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071291 DataRate::BitsPerSec(kTargetBitrateBps),
1292 DataRate::BitsPerSec(kTargetBitrateBps),
1293 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401294
perkja49cbd32016-09-16 14:53:411295 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521296 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401297
mflodmancc3d4422017-08-03 15:27:511298 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401299 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 07:43:501300 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411301 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1302 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401303}
1304
mflodmancc3d4422017-08-03 15:27:511305TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 13:14:011306 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071307 DataRate::BitsPerSec(kTargetBitrateBps),
1308 DataRate::BitsPerSec(kTargetBitrateBps),
1309 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 08:17:401310
1311 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 14:53:411312 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521313 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401314 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1315 // call to ContinueEncode.
perkja49cbd32016-09-16 14:53:411316 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1317 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401318 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 14:06:521319 WaitForEncodedFrame(3);
perkj26091b12016-09-01 08:17:401320
mflodmancc3d4422017-08-03 15:27:511321 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401322}
1323
Noah Richards51db4212019-06-12 13:59:121324TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1325 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071326 DataRate::BitsPerSec(kTargetBitrateBps),
1327 DataRate::BitsPerSec(kTargetBitrateBps),
1328 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121329
1330 rtc::Event frame_destroyed_event;
1331 video_source_.IncomingCapturedFrame(
1332 CreateFakeNativeFrame(1, &frame_destroyed_event));
1333 ExpectDroppedFrame();
1334 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1335 video_stream_encoder_->Stop();
1336}
1337
1338TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1339 // Use the cropping factory.
1340 video_encoder_config_.video_stream_factory =
1341 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1342 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1343 kMaxPayloadLength);
1344 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1345
1346 // Capture a frame at codec_width_/codec_height_.
1347 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071348 DataRate::BitsPerSec(kTargetBitrateBps),
1349 DataRate::BitsPerSec(kTargetBitrateBps),
1350 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 13:59:121351 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1352 WaitForEncodedFrame(1);
1353 // The encoder will have been configured once.
1354 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1355 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1356 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1357
1358 // Now send in a fake frame that needs to be cropped as the width/height
1359 // aren't divisible by 4 (see CreateEncoderStreams above).
1360 rtc::Event frame_destroyed_event;
1361 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1362 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1363 ExpectDroppedFrame();
1364 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1365 video_stream_encoder_->Stop();
1366}
1367
Ying Wang9b881ab2020-02-07 13:29:321368TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
1369 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071370 DataRate::BitsPerSec(kTargetBitrateBps),
1371 DataRate::BitsPerSec(kTargetBitrateBps),
1372 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 13:29:321373 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1374 WaitForEncodedFrame(1);
1375
1376 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071377 DataRate::BitsPerSec(kTargetBitrateBps),
1378 DataRate::BitsPerSec(kTargetBitrateBps),
1379 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 13:29:321380 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1381 // frames. Adding two frames means that the first frame will be dropped and
1382 // the second frame will be sent to the encoder.
1383 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1384 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1385 WaitForEncodedFrame(3);
1386 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1387 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1388 WaitForEncodedFrame(5);
1389 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1390 video_stream_encoder_->Stop();
1391}
1392
mflodmancc3d4422017-08-03 15:27:511393TEST_F(VideoStreamEncoderTest,
1394 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 13:14:011395 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071396 DataRate::BitsPerSec(kTargetBitrateBps),
1397 DataRate::BitsPerSec(kTargetBitrateBps),
1398 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 20:37:571399 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061400
1401 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551402 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521403 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571404 // The encoder will have been configured once when the first frame is
1405 // received.
1406 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061407
1408 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:511409 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 13:52:061410 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 15:27:511411 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:471412 kMaxPayloadLength);
Per512ecb32016-09-23 13:52:061413
1414 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551415 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521416 WaitForEncodedFrame(2);
Per21d45d22016-10-30 20:37:571417 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-30 06:25:401418 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-30 05:39:101419
mflodmancc3d4422017-08-03 15:27:511420 video_stream_encoder_->Stop();
perkj26105b42016-09-30 05:39:101421}
1422
mflodmancc3d4422017-08-03 15:27:511423TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 13:14:011424 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071425 DataRate::BitsPerSec(kTargetBitrateBps),
1426 DataRate::BitsPerSec(kTargetBitrateBps),
1427 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-03 06:45:261428
1429 // Capture a frame and wait for it to synchronize with the encoder thread.
1430 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521431 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571432 // The encoder will have been configured once.
1433 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261434 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1435 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1436
1437 codec_width_ *= 2;
1438 codec_height_ *= 2;
1439 // Capture a frame with a higher resolution and wait for it to synchronize
1440 // with the encoder thread.
1441 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521442 WaitForEncodedFrame(2);
perkjfa10b552016-10-03 06:45:261443 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1444 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 20:37:571445 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261446
mflodmancc3d4422017-08-03 15:27:511447 video_stream_encoder_->Stop();
perkjfa10b552016-10-03 06:45:261448}
1449
Sergey Silkin443b7ee2019-06-28 10:53:071450TEST_F(VideoStreamEncoderTest,
1451 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1452 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071453 DataRate::BitsPerSec(kTargetBitrateBps),
1454 DataRate::BitsPerSec(kTargetBitrateBps),
1455 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 10:53:071456
1457 // Capture a frame and wait for it to synchronize with the encoder thread.
1458 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1459 WaitForEncodedFrame(1);
1460
1461 VideoEncoderConfig video_encoder_config;
1462 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1463 // Changing the max payload data length recreates encoder.
1464 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1465 kMaxPayloadLength / 2);
1466
1467 // Capture a frame and wait for it to synchronize with the encoder thread.
1468 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1469 WaitForEncodedFrame(2);
1470 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1471
1472 video_stream_encoder_->Stop();
1473}
1474
Sergey Silkin5ee69672019-07-02 12:18:341475TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1476 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071477 DataRate::BitsPerSec(kTargetBitrateBps),
1478 DataRate::BitsPerSec(kTargetBitrateBps),
1479 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 12:18:341480
1481 VideoEncoderConfig video_encoder_config;
1482 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1483 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1484 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1485 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1486 kMaxPayloadLength);
1487
1488 // Capture a frame and wait for it to synchronize with the encoder thread.
1489 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1490 WaitForEncodedFrame(1);
1491 // The encoder will have been configured once when the first frame is
1492 // received.
1493 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1494 EXPECT_EQ(kTargetBitrateBps,
1495 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1496 EXPECT_EQ(kStartBitrateBps,
1497 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1498
Sergey Silkin6456e352019-07-08 15:56:401499 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1500 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 12:18:341501 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1502 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1503 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1504 kMaxPayloadLength);
1505
1506 // Capture a frame and wait for it to synchronize with the encoder thread.
1507 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1508 WaitForEncodedFrame(2);
1509 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1510 // Bitrate limits have changed - rate allocator should be reconfigured,
1511 // encoder should not be reconfigured.
1512 EXPECT_EQ(kTargetBitrateBps * 2,
1513 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1514 EXPECT_EQ(kStartBitrateBps * 2,
1515 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1516 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1517
1518 video_stream_encoder_->Stop();
1519}
1520
Sergey Silkin6456e352019-07-08 15:56:401521TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 13:48:401522 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Sergey Silkin6456e352019-07-08 15:56:401523 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071524 DataRate::BitsPerSec(kTargetBitrateBps),
1525 DataRate::BitsPerSec(kTargetBitrateBps),
1526 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401527
Sergey Silkincd02eba2020-01-20 13:48:401528 const uint32_t kMinEncBitrateKbps = 100;
1529 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051530 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 13:48:401531 /*frame_size_pixels=*/codec_width_ * codec_height_,
1532 /*min_start_bitrate_bps=*/0,
1533 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1534 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051535 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1536
Sergey Silkincd02eba2020-01-20 13:48:401537 VideoEncoderConfig video_encoder_config;
1538 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1539 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1540 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1541 (kMinEncBitrateKbps + 1) * 1000;
1542 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1543 kMaxPayloadLength);
1544
1545 // When both encoder and app provide bitrate limits, the intersection of
1546 // provided sets should be used.
1547 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1548 WaitForEncodedFrame(1);
1549 EXPECT_EQ(kMaxEncBitrateKbps,
1550 bitrate_allocator_factory_.codec_config().maxBitrate);
1551 EXPECT_EQ(kMinEncBitrateKbps + 1,
1552 bitrate_allocator_factory_.codec_config().minBitrate);
1553
1554 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1555 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1556 (kMinEncBitrateKbps - 1) * 1000;
1557 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1558 kMaxPayloadLength);
1559 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 14:04:051560 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 13:48:401561 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 14:04:051562 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:401563 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051564 bitrate_allocator_factory_.codec_config().minBitrate);
1565
Sergey Silkincd02eba2020-01-20 13:48:401566 video_stream_encoder_->Stop();
1567}
1568
1569TEST_F(VideoStreamEncoderTest,
1570 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
1571 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071572 DataRate::BitsPerSec(kTargetBitrateBps),
1573 DataRate::BitsPerSec(kTargetBitrateBps),
1574 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 13:48:401575
1576 const uint32_t kMinAppBitrateKbps = 100;
1577 const uint32_t kMaxAppBitrateKbps = 200;
1578 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1579 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1580 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1581 /*frame_size_pixels=*/codec_width_ * codec_height_,
1582 /*min_start_bitrate_bps=*/0,
1583 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1584 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1585 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1586
1587 VideoEncoderConfig video_encoder_config;
1588 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1589 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1590 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1591 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 14:04:051592 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1593 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 14:04:051594
Sergey Silkincd02eba2020-01-20 13:48:401595 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1596 WaitForEncodedFrame(1);
1597 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051598 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 13:48:401599 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 14:04:051600 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 15:56:401601
1602 video_stream_encoder_->Stop();
1603}
1604
1605TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 14:04:051606 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 15:56:401607 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071608 DataRate::BitsPerSec(kTargetBitrateBps),
1609 DataRate::BitsPerSec(kTargetBitrateBps),
1610 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401611
1612 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 14:04:051613 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 15:56:401614 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 14:04:051615 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 15:56:401616 fake_encoder_.SetResolutionBitrateLimits(
1617 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1618
1619 VideoEncoderConfig video_encoder_config;
1620 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1621 video_encoder_config.max_bitrate_bps = 0;
1622 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1623 kMaxPayloadLength);
1624
Sergey Silkin6b2cec12019-08-09 14:04:051625 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 15:56:401626 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1627 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 14:04:051628 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1629 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401630 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1631 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1632
Sergey Silkin6b2cec12019-08-09 14:04:051633 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 15:56:401634 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1635 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 14:04:051636 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1637 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401638 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1639 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1640
Sergey Silkin6b2cec12019-08-09 14:04:051641 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 15:56:401642 // encoder for 360p should be used.
1643 video_source_.IncomingCapturedFrame(
1644 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1645 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 14:04:051646 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1647 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401648 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1649 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1650
Sergey Silkin6b2cec12019-08-09 14:04:051651 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 15:56:401652 // ignored.
1653 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1654 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 14:04:051655 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1656 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401657 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1658 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051659 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1660 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401661 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1662 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1663
1664 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1665 // for 270p should be used.
1666 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1667 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 14:04:051668 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1669 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401670 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1671 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1672
1673 video_stream_encoder_->Stop();
1674}
1675
Sergey Silkin6b2cec12019-08-09 14:04:051676TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1677 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071678 DataRate::BitsPerSec(kTargetBitrateBps),
1679 DataRate::BitsPerSec(kTargetBitrateBps),
1680 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 14:04:051681
1682 VideoEncoderConfig video_encoder_config;
1683 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1684 video_encoder_config.max_bitrate_bps = 0;
1685 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1686 kMaxPayloadLength);
1687
1688 // Encode 720p frame to get the default encoder target bitrate.
1689 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1690 WaitForEncodedFrame(1);
1691 const uint32_t kDefaultTargetBitrateFor720pKbps =
1692 bitrate_allocator_factory_.codec_config()
1693 .simulcastStream[0]
1694 .targetBitrate;
1695
1696 // Set the max recommended encoder bitrate to something lower than the default
1697 // target bitrate.
1698 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1699 1280 * 720, 10 * 1000, 10 * 1000,
1700 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1701 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1702
1703 // Change resolution to trigger encoder reinitialization.
1704 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1705 WaitForEncodedFrame(2);
1706 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1707 WaitForEncodedFrame(3);
1708
1709 // Ensure the target bitrate is capped by the max bitrate.
1710 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1711 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1712 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1713 .simulcastStream[0]
1714 .targetBitrate *
1715 1000,
1716 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1717
1718 video_stream_encoder_->Stop();
1719}
1720
mflodmancc3d4422017-08-03 15:27:511721TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 18:45:461722 EXPECT_TRUE(video_source_.has_sinks());
1723 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:511724 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411725 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:461726 EXPECT_FALSE(video_source_.has_sinks());
1727 EXPECT_TRUE(new_video_source.has_sinks());
1728
mflodmancc3d4422017-08-03 15:27:511729 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:461730}
1731
mflodmancc3d4422017-08-03 15:27:511732TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 18:45:461733 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:511734 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 18:45:461735 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:511736 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:461737}
1738
Rasmus Brandt5cad55b2019-12-19 08:47:111739TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1740 constexpr int kRequestedResolutionAlignment = 7;
1741 video_source_.set_adaptation_enabled(true);
1742 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
1743 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071744 DataRate::BitsPerSec(kTargetBitrateBps),
1745 DataRate::BitsPerSec(kTargetBitrateBps),
1746 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 08:47:111747
1748 // On the 1st frame, we should have initialized the encoder and
1749 // asked for its resolution requirements.
1750 video_source_.IncomingCapturedFrame(
1751 CreateFrame(1, codec_width_, codec_height_));
1752 WaitForEncodedFrame(1);
1753 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1754 kRequestedResolutionAlignment);
1755
1756 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1757 // (It's up the to the encoder to potentially drop the previous frame,
1758 // to avoid coding back-to-back keyframes.)
1759 video_source_.IncomingCapturedFrame(
1760 CreateFrame(2, codec_width_, codec_height_));
1761 WaitForEncodedFrame(2);
1762 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1763
1764 video_stream_encoder_->Stop();
1765}
1766
Jonathan Yubc771b72017-12-09 01:04:291767TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1768 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-14 06:25:221769 const int kWidth = 1280;
1770 const int kHeight = 720;
Jonathan Yubc771b72017-12-09 01:04:291771
1772 // We rely on the automatic resolution adaptation, but we handle framerate
1773 // adaptation manually by mocking the stats proxy.
1774 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-14 06:25:221775
Taylor Brandstetter49fcc102018-05-16 21:20:411776 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 13:14:011777 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071778 DataRate::BitsPerSec(kTargetBitrateBps),
1779 DataRate::BitsPerSec(kTargetBitrateBps),
1780 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 21:20:411781 video_stream_encoder_->SetSource(&video_source_,
1782 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-09 01:04:291783 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-14 06:25:221784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:291785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:221786 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1787
Jonathan Yubc771b72017-12-09 01:04:291788 // Adapt down as far as possible.
1789 rtc::VideoSinkWants last_wants;
1790 int64_t t = 1;
1791 int loop_count = 0;
1792 do {
1793 ++loop_count;
1794 last_wants = video_source_.sink_wants();
1795
1796 // Simulate the framerate we've been asked to adapt to.
1797 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1798 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1799 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1800 mock_stats.input_frame_rate = fps;
1801 stats_proxy_->SetMockStats(mock_stats);
1802
1803 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1804 sink_.WaitForEncodedFrame(t);
1805 t += frame_interval_ms;
1806
mflodmancc3d4422017-08-03 15:27:511807 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:291808 VerifyBalancedModeFpsRange(
1809 video_source_.sink_wants(),
1810 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1811 } while (video_source_.sink_wants().max_pixel_count <
1812 last_wants.max_pixel_count ||
1813 video_source_.sink_wants().max_framerate_fps <
1814 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:221815
Jonathan Yubc771b72017-12-09 01:04:291816 // Verify that we've adapted all the way down.
1817 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:221818 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:291819 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1820 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-14 06:25:221821 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:291822 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1823 *video_source_.last_sent_height());
1824 EXPECT_EQ(kMinBalancedFramerateFps,
1825 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:221826
Jonathan Yubc771b72017-12-09 01:04:291827 // Adapt back up the same number of times we adapted down.
1828 for (int i = 0; i < loop_count - 1; ++i) {
1829 last_wants = video_source_.sink_wants();
1830
1831 // Simulate the framerate we've been asked to adapt to.
1832 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1833 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1834 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1835 mock_stats.input_frame_rate = fps;
1836 stats_proxy_->SetMockStats(mock_stats);
1837
1838 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1839 sink_.WaitForEncodedFrame(t);
1840 t += frame_interval_ms;
1841
mflodmancc3d4422017-08-03 15:27:511842 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-09 01:04:291843 VerifyBalancedModeFpsRange(
1844 video_source_.sink_wants(),
1845 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1846 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1847 last_wants.max_pixel_count ||
1848 video_source_.sink_wants().max_framerate_fps >
1849 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:221850 }
1851
Åsa Persson8c1bf952018-09-13 08:42:191852 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-09 01:04:291853 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:221854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:291855 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1856 EXPECT_EQ((loop_count - 1) * 2,
1857 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:221858
mflodmancc3d4422017-08-03 15:27:511859 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:221860}
Rasmus Brandt5cad55b2019-12-19 08:47:111861
mflodmancc3d4422017-08-03 15:27:511862TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 13:14:011863 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071864 DataRate::BitsPerSec(kTargetBitrateBps),
1865 DataRate::BitsPerSec(kTargetBitrateBps),
1866 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:521867 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 18:45:461868
sprangc5d62e22017-04-03 06:53:041869 const int kFrameWidth = 1280;
1870 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:041871
Åsa Persson8c1bf952018-09-13 08:42:191872 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 18:45:461873
kthelgason5e13d412016-12-01 11:59:511874 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:041875 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:521876 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:041877 frame_timestamp += kFrameIntervalMs;
1878
perkj803d97f2016-11-01 18:45:461879 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:511880 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:461881 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:041882 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:521883 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:041884 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 14:23:481885
asapersson0944a802017-04-07 07:57:581886 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-03 06:53:041887 // wanted resolution.
1888 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1889 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1890 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 08:42:191891 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:041892
1893 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 18:45:461894 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:511895 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411896 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:121897 // Give the encoder queue time to process the change in degradation preference
1898 // by waiting for an encoded frame.
1899 new_video_source.IncomingCapturedFrame(
1900 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1901 sink_.WaitForEncodedFrame(frame_timestamp);
1902 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:041903 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 08:42:191904 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 18:45:461905
sprangc5d62e22017-04-03 06:53:041906 // Force an input frame rate to be available, or the adaptation call won't
1907 // know what framerate to adapt form.
asapersson02465b82017-04-10 08:12:521908 const int kInputFps = 30;
sprangc5d62e22017-04-03 06:53:041909 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 08:12:521910 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-03 06:53:041911 stats_proxy_->SetMockStats(stats);
1912
mflodmancc3d4422017-08-03 15:27:511913 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:461914 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:041915 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:521916 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:041917 frame_timestamp += kFrameIntervalMs;
1918
1919 // Some framerate constraint should be set.
sprang84a37592017-02-10 15:04:271920 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-03 06:53:041921 EXPECT_EQ(std::numeric_limits<int>::max(),
1922 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:521923 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-03 06:53:041924
asapersson02465b82017-04-10 08:12:521925 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 21:20:411926 video_stream_encoder_->SetSource(&new_video_source,
1927 webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 10:42:121928 // Give the encoder queue time to process the change in degradation preference
1929 // by waiting for an encoded frame.
1930 new_video_source.IncomingCapturedFrame(
1931 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1932 sink_.WaitForEncodedFrame(frame_timestamp);
1933 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 08:42:191934 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-03 06:53:041935
mflodmancc3d4422017-08-03 15:27:511936 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:041937 new_video_source.IncomingCapturedFrame(
1938 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:521939 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:041940 frame_timestamp += kFrameIntervalMs;
1941
1942 // Still no degradation.
Åsa Persson8c1bf952018-09-13 08:42:191943 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 18:45:461944
1945 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 15:27:511946 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411947 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 10:42:121948 // Give the encoder queue time to process the change in degradation preference
1949 // by waiting for an encoded frame.
1950 new_video_source.IncomingCapturedFrame(
1951 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1952 sink_.WaitForEncodedFrame(frame_timestamp);
1953 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:041954 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1955 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 15:04:271956 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 08:42:191957 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:041958
1959 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 15:27:511960 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411961 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:121962 // Give the encoder queue time to process the change in degradation preference
1963 // by waiting for an encoded frame.
1964 new_video_source.IncomingCapturedFrame(
1965 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1966 sink_.WaitForEncodedFrame(frame_timestamp);
1967 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:041968 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1969 EXPECT_EQ(std::numeric_limits<int>::max(),
1970 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:521971 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 18:45:461972
mflodmancc3d4422017-08-03 15:27:511973 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:461974}
1975
mflodmancc3d4422017-08-03 15:27:511976TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 13:14:011977 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:071978 DataRate::BitsPerSec(kTargetBitrateBps),
1979 DataRate::BitsPerSec(kTargetBitrateBps),
1980 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 18:45:461981
asaperssonfab67072017-04-04 12:51:491982 const int kWidth = 1280;
1983 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:491984 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521985 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 12:51:491986 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1987 EXPECT_FALSE(stats.bw_limited_resolution);
1988 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1989
1990 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:511991 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:491992 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521993 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 12:51:491994
1995 stats = stats_proxy_->GetStats();
1996 EXPECT_TRUE(stats.bw_limited_resolution);
1997 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1998
1999 // Trigger adapt up.
mflodmancc3d4422017-08-03 15:27:512000 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:492001 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522002 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 12:51:492003
2004 stats = stats_proxy_->GetStats();
2005 EXPECT_FALSE(stats.bw_limited_resolution);
2006 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2007 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2008
mflodmancc3d4422017-08-03 15:27:512009 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 12:51:492010}
2011
mflodmancc3d4422017-08-03 15:27:512012TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 13:14:012013 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072014 DataRate::BitsPerSec(kTargetBitrateBps),
2015 DataRate::BitsPerSec(kTargetBitrateBps),
2016 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:492017
2018 const int kWidth = 1280;
2019 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:492020 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522021 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 18:45:462022 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2023 EXPECT_FALSE(stats.cpu_limited_resolution);
2024 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2025
2026 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512027 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492028 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522029 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 18:45:462030
2031 stats = stats_proxy_->GetStats();
2032 EXPECT_TRUE(stats.cpu_limited_resolution);
2033 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2034
2035 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 15:27:512036 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 12:51:492037 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522038 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 18:45:462039
2040 stats = stats_proxy_->GetStats();
2041 EXPECT_FALSE(stats.cpu_limited_resolution);
2042 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 12:51:492043 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:462044
mflodmancc3d4422017-08-03 15:27:512045 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462046}
2047
mflodmancc3d4422017-08-03 15:27:512048TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 13:14:012049 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072050 DataRate::BitsPerSec(kTargetBitrateBps),
2051 DataRate::BitsPerSec(kTargetBitrateBps),
2052 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112053
asaperssonfab67072017-04-04 12:51:492054 const int kWidth = 1280;
2055 const int kHeight = 720;
2056 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522057 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112058 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182059 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112060 EXPECT_FALSE(stats.cpu_limited_resolution);
2061 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2062
asaperssonfab67072017-04-04 12:51:492063 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:512064 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492065 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522066 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112067 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182068 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112069 EXPECT_TRUE(stats.cpu_limited_resolution);
2070 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2071
2072 // Set new source with adaptation still enabled.
2073 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512074 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412075 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112076
asaperssonfab67072017-04-04 12:51:492077 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522078 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112079 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182080 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112081 EXPECT_TRUE(stats.cpu_limited_resolution);
2082 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2083
2084 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412085 video_stream_encoder_->SetSource(&new_video_source,
2086 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 09:44:112087
asaperssonfab67072017-04-04 12:51:492088 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522089 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112090 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182091 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112092 EXPECT_FALSE(stats.cpu_limited_resolution);
2093 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2094
2095 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 15:27:512096 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412097 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:112098
asaperssonfab67072017-04-04 12:51:492099 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522100 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112101 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182102 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112103 EXPECT_TRUE(stats.cpu_limited_resolution);
2104 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2105
asaperssonfab67072017-04-04 12:51:492106 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 15:27:512107 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 12:51:492108 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522109 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 09:44:112110 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182111 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:112112 EXPECT_FALSE(stats.cpu_limited_resolution);
2113 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:522114 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112115
mflodmancc3d4422017-08-03 15:27:512116 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112117}
2118
mflodmancc3d4422017-08-03 15:27:512119TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 13:14:012120 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072121 DataRate::BitsPerSec(kTargetBitrateBps),
2122 DataRate::BitsPerSec(kTargetBitrateBps),
2123 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112124
asaperssonfab67072017-04-04 12:51:492125 const int kWidth = 1280;
2126 const int kHeight = 720;
2127 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522128 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112129 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112130 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022131 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492132 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112133
2134 // Set new source with adaptation still enabled.
2135 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 21:20:412136 video_stream_encoder_->SetSource(&new_video_source,
2137 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112138
asaperssonfab67072017-04-04 12:51:492139 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522140 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:112141 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112142 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022143 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492144 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112145
asaperssonfab67072017-04-04 12:51:492146 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512147 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492148 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522149 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112150 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112151 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022152 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492153 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112154
asaperssonfab67072017-04-04 12:51:492155 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 21:20:412156 video_stream_encoder_->SetSource(&new_video_source,
2157 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:112158
asaperssonfab67072017-04-04 12:51:492159 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522160 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112161 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112162 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022163 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492164 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:112165
asapersson02465b82017-04-10 08:12:522166 // Disable resolution scaling.
mflodmancc3d4422017-08-03 15:27:512167 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412168 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:112169
asaperssonfab67072017-04-04 12:51:492170 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522171 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112172 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112173 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022174 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492175 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2176 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 09:44:112177
mflodmancc3d4422017-08-03 15:27:512178 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112179}
2180
mflodmancc3d4422017-08-03 15:27:512181TEST_F(VideoStreamEncoderTest,
2182 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 13:14:012183 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072184 DataRate::BitsPerSec(kTargetBitrateBps),
2185 DataRate::BitsPerSec(kTargetBitrateBps),
2186 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 12:29:122187
2188 const int kWidth = 1280;
2189 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 08:42:192190 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 12:29:122191 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 08:42:192192 video_source_.IncomingCapturedFrame(
2193 CreateFrame(timestamp_ms, kWidth, kHeight));
2194 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122195 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2196 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2197 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2198
2199 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512200 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:192201 timestamp_ms += kFrameIntervalMs;
2202 video_source_.IncomingCapturedFrame(
2203 CreateFrame(timestamp_ms, kWidth, kHeight));
2204 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122205 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2206 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2207 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2208
2209 // Trigger overuse.
mflodmancc3d4422017-08-03 15:27:512210 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:192211 timestamp_ms += kFrameIntervalMs;
2212 video_source_.IncomingCapturedFrame(
2213 CreateFrame(timestamp_ms, kWidth, kHeight));
2214 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122215 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2216 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2217 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2218
Niels Möller4db138e2018-04-19 07:04:132219 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 12:29:122220 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:132221
2222 VideoEncoderConfig video_encoder_config;
2223 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2224 // Make format different, to force recreation of encoder.
2225 video_encoder_config.video_format.parameters["foo"] = "foo";
2226 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:472227 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 08:42:192228 timestamp_ms += kFrameIntervalMs;
2229 video_source_.IncomingCapturedFrame(
2230 CreateFrame(timestamp_ms, kWidth, kHeight));
2231 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122232 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2233 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2234 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2235
mflodmancc3d4422017-08-03 15:27:512236 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 12:29:122237}
2238
mflodmancc3d4422017-08-03 15:27:512239TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 17:39:322240 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
2241 video_stream_encoder_->OnBitrateUpdated(
2242 DataRate::BitsPerSec(kTargetBitrateBps),
2243 DataRate::BitsPerSec(kTargetBitrateBps),
2244 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2245
2246 const int kWidth = 1280;
2247 const int kHeight = 720;
2248 int sequence = 1;
2249
2250 // Enable BALANCED preference, no initial limitation.
2251 test::FrameForwarder source;
2252 video_stream_encoder_->SetSource(&source,
2253 webrtc::DegradationPreference::BALANCED);
2254 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2255 WaitForEncodedFrame(sequence++);
2256 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2257 EXPECT_FALSE(stats.cpu_limited_resolution);
2258 EXPECT_FALSE(stats.cpu_limited_framerate);
2259 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2260
2261 // Trigger CPU overuse, should now adapt down.
2262 video_stream_encoder_->TriggerCpuOveruse();
2263 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2264 WaitForEncodedFrame(sequence++);
2265 stats = stats_proxy_->GetStats();
2266 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2267
2268 // Set new degradation preference should clear restrictions since we changed
2269 // from BALANCED.
2270 video_stream_encoder_->SetSource(
2271 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2272 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2273 WaitForEncodedFrame(sequence++);
2274 stats = stats_proxy_->GetStats();
2275 EXPECT_FALSE(stats.cpu_limited_resolution);
2276 EXPECT_FALSE(stats.cpu_limited_framerate);
2277 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2278
2279 // Force an input frame rate to be available, or the adaptation call won't
2280 // know what framerate to adapt from.
2281 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2282 mock_stats.input_frame_rate = 30;
2283 stats_proxy_->SetMockStats(mock_stats);
2284 video_stream_encoder_->TriggerCpuOveruse();
2285 stats_proxy_->ResetMockStats();
2286 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2287 WaitForEncodedFrame(sequence++);
2288
2289 // We have now adapted once.
2290 stats = stats_proxy_->GetStats();
2291 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2292
2293 // Back to BALANCED, should clear the restrictions again.
2294 video_stream_encoder_->SetSource(&source,
2295 webrtc::DegradationPreference::BALANCED);
2296 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2297 WaitForEncodedFrame(sequence++);
2298 stats = stats_proxy_->GetStats();
2299 EXPECT_FALSE(stats.cpu_limited_resolution);
2300 EXPECT_FALSE(stats.cpu_limited_framerate);
2301 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2302
2303 video_stream_encoder_->Stop();
2304}
2305
2306TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 15:27:512307 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 13:14:012308 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072309 DataRate::BitsPerSec(kTargetBitrateBps),
2310 DataRate::BitsPerSec(kTargetBitrateBps),
2311 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 18:45:462312
asapersson0944a802017-04-07 07:57:582313 const int kWidth = 1280;
2314 const int kHeight = 720;
sprang84a37592017-02-10 15:04:272315 int sequence = 1;
perkj803d97f2016-11-01 18:45:462316
asaperssonfab67072017-04-04 12:51:492317 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522318 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462319 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 15:04:272320 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022321 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 15:04:272322 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2323
asapersson02465b82017-04-10 08:12:522324 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 15:27:512325 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492326 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522327 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 15:04:272328 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 18:45:462329 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022330 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:462331 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2332
2333 // Set new source with adaptation still enabled.
2334 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512335 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412336 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:462337
2338 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492339 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522340 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462341 stats = stats_proxy_->GetStats();
2342 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:182343 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:462344 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2345
sprangc5d62e22017-04-03 06:53:042346 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 15:27:512347 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412348 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 18:45:462349 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492350 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522351 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462352 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-03 06:53:042353 // Not adapted at first.
perkj803d97f2016-11-01 18:45:462354 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:182355 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:462356 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2357
sprangc5d62e22017-04-03 06:53:042358 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-16 06:40:182359 // know what framerate to adapt from.
sprangc5d62e22017-04-03 06:53:042360 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2361 mock_stats.input_frame_rate = 30;
2362 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:512363 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042364 stats_proxy_->ResetMockStats();
2365
2366 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492367 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522368 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:042369
2370 // Framerate now adapted.
2371 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182372 EXPECT_FALSE(stats.cpu_limited_resolution);
2373 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042374 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2375
2376 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 21:20:412377 video_stream_encoder_->SetSource(&new_video_source,
2378 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-03 06:53:042379 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492380 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522381 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:042382
2383 stats = stats_proxy_->GetStats();
2384 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022385 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042386 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2387
2388 // Try to trigger overuse. Should not succeed.
2389 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:512390 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042391 stats_proxy_->ResetMockStats();
2392
2393 stats = stats_proxy_->GetStats();
2394 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022395 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042396 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2397
2398 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 15:27:512399 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412400 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 12:51:492401 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522402 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462403 stats = stats_proxy_->GetStats();
2404 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022405 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042406 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 18:45:462407
2408 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 15:27:512409 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 12:51:492410 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522411 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462412 stats = stats_proxy_->GetStats();
2413 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022414 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042415 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2416
2417 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 15:27:512418 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412419 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:042420 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492421 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522422 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:042423 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 07:01:022424 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-03 06:53:042425 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022426 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042427 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2428
2429 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 15:27:512430 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-03 06:53:042431 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492432 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522433 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:042434 stats = stats_proxy_->GetStats();
2435 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022436 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042437 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:522438 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:462439
mflodmancc3d4422017-08-03 15:27:512440 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462441}
2442
mflodmancc3d4422017-08-03 15:27:512443TEST_F(VideoStreamEncoderTest,
2444 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 12:51:492445 const int kWidth = 1280;
2446 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012447 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072448 DataRate::BitsPerSec(kTargetBitrateBps),
2449 DataRate::BitsPerSec(kTargetBitrateBps),
2450 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 09:44:112451
asaperssonfab67072017-04-04 12:51:492452 // Expect no scaling to begin with.
asapersson02465b82017-04-10 08:12:522453 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 09:44:112454
asaperssonfab67072017-04-04 12:51:492455 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522456 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112457
asaperssonfab67072017-04-04 12:51:492458 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:512459 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 11:59:512460
asaperssonfab67072017-04-04 12:51:492461 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522462 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 11:59:512463
kthelgason876222f2016-11-29 09:44:112464 // Expect a scale down.
2465 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 12:51:492466 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 09:44:112467
asapersson02465b82017-04-10 08:12:522468 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 09:44:112469 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512470 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412471 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:112472
asaperssonfab67072017-04-04 12:51:492473 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:512474 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492475 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522476 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112477
asaperssonfab67072017-04-04 12:51:492478 // Expect no scaling.
sprangc5d62e22017-04-03 06:53:042479 EXPECT_EQ(std::numeric_limits<int>::max(),
2480 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:112481
asaperssonfab67072017-04-04 12:51:492482 // Trigger scale up.
mflodmancc3d4422017-08-03 15:27:512483 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:492484 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522485 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112486
asapersson02465b82017-04-10 08:12:522487 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-03 06:53:042488 EXPECT_EQ(std::numeric_limits<int>::max(),
2489 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:112490
mflodmancc3d4422017-08-03 15:27:512491 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112492}
2493
mflodmancc3d4422017-08-03 15:27:512494TEST_F(VideoStreamEncoderTest,
2495 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:522496 const int kWidth = 1280;
2497 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012498 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072499 DataRate::BitsPerSec(kTargetBitrateBps),
2500 DataRate::BitsPerSec(kTargetBitrateBps),
2501 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:522502
Taylor Brandstetter49fcc102018-05-16 21:20:412503 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:522504 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:512505 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412506 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:522507
2508 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522509 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 08:42:192510 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522511 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2512 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2513
2514 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512515 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-16 06:40:182516 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 08:12:522517 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2518 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2519 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2520
2521 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 15:27:512522 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 08:12:522523 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2524 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2525 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2526
mflodmancc3d4422017-08-03 15:27:512527 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:522528}
2529
mflodmancc3d4422017-08-03 15:27:512530TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:222531 const int kWidth = 1280;
2532 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012533 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072534 DataRate::BitsPerSec(kTargetBitrateBps),
2535 DataRate::BitsPerSec(kTargetBitrateBps),
2536 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:222537
Taylor Brandstetter49fcc102018-05-16 21:20:412538 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:222539 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:412540 video_stream_encoder_->SetSource(&source,
2541 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:222542 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2543 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 08:42:192544 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222545
2546 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512547 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:222548 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2550 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2551 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2552
2553 // Trigger adapt down for same input resolution, expect no change.
2554 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2555 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:512556 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:222557 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2558 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2559 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2560
2561 // Trigger adapt down for larger input resolution, expect no change.
2562 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2563 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:512564 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:222565 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2567 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2568
mflodmancc3d4422017-08-03 15:27:512569 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222570}
2571
mflodmancc3d4422017-08-03 15:27:512572TEST_F(VideoStreamEncoderTest,
2573 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:522574 const int kWidth = 1280;
2575 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012576 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072577 DataRate::BitsPerSec(kTargetBitrateBps),
2578 DataRate::BitsPerSec(kTargetBitrateBps),
2579 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:522580
Taylor Brandstetter49fcc102018-05-16 21:20:412581 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:522582 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:512583 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412584 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:522585
2586 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522587 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192588 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522589 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2590 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2591
2592 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:512593 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:192594 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522595 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2596 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2597
mflodmancc3d4422017-08-03 15:27:512598 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:522599}
2600
mflodmancc3d4422017-08-03 15:27:512601TEST_F(VideoStreamEncoderTest,
2602 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 08:12:522603 const int kWidth = 1280;
2604 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012605 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072606 DataRate::BitsPerSec(kTargetBitrateBps),
2607 DataRate::BitsPerSec(kTargetBitrateBps),
2608 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:522609
Taylor Brandstetter49fcc102018-05-16 21:20:412610 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 08:12:522611 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:512612 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412613 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 08:12:522614
2615 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522616 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192617 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-16 06:40:182618 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:522619 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2620
2621 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:512622 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:192623 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-16 06:40:182624 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:522625 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2626
mflodmancc3d4422017-08-03 15:27:512627 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:522628}
2629
mflodmancc3d4422017-08-03 15:27:512630TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:222631 const int kWidth = 1280;
2632 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012633 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072634 DataRate::BitsPerSec(kTargetBitrateBps),
2635 DataRate::BitsPerSec(kTargetBitrateBps),
2636 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:222637
Taylor Brandstetter49fcc102018-05-16 21:20:412638 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:222639 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:412640 video_stream_encoder_->SetSource(&source,
2641 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:222642
2643 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2644 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192645 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222646 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2647 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2648 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2649
2650 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:512651 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192652 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2654 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2655 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2656
mflodmancc3d4422017-08-03 15:27:512657 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222658}
2659
mflodmancc3d4422017-08-03 15:27:512660TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-16 06:40:182661 const int kWidth = 1280;
2662 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012663 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072664 DataRate::BitsPerSec(kTargetBitrateBps),
2665 DataRate::BitsPerSec(kTargetBitrateBps),
2666 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:182667
Taylor Brandstetter49fcc102018-05-16 21:20:412668 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-16 06:40:182669 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:412670 video_stream_encoder_->SetSource(&source,
2671 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-16 06:40:182672
2673 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2674 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192675 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-16 06:40:182676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2678 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2679
2680 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:512681 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192682 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-16 06:40:182683 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2684 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2685 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2686
mflodmancc3d4422017-08-03 15:27:512687 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:182688}
2689
mflodmancc3d4422017-08-03 15:27:512690TEST_F(VideoStreamEncoderTest,
2691 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:522692 const int kWidth = 1280;
2693 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012694 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072695 DataRate::BitsPerSec(kTargetBitrateBps),
2696 DataRate::BitsPerSec(kTargetBitrateBps),
2697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 08:12:522698
Taylor Brandstetter49fcc102018-05-16 21:20:412699 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:522700 AdaptingFrameForwarder source;
2701 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:512702 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412703 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:522704
2705 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522706 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 08:42:192707 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522708 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2709 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2710
2711 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512712 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 08:12:522713 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522714 WaitForEncodedFrame(2);
asapersson09f05612017-05-16 06:40:182715 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 08:12:522716 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2717 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2718
2719 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512720 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192721 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2723 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2725
mflodmancc3d4422017-08-03 15:27:512726 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:522727}
2728
mflodmancc3d4422017-08-03 15:27:512729TEST_F(VideoStreamEncoderTest,
2730 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-16 06:40:182731 const int kWidth = 1280;
2732 const int kHeight = 720;
2733 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 13:14:012734 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072735 DataRate::BitsPerSec(kTargetBitrateBps),
2736 DataRate::BitsPerSec(kTargetBitrateBps),
2737 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:182738
2739 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2740 stats.input_frame_rate = kInputFps;
2741 stats_proxy_->SetMockStats(stats);
2742
Taylor Brandstetter49fcc102018-05-16 21:20:412743 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-16 06:40:182744 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2745 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 08:42:192746 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-16 06:40:182747
2748 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512749 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-16 06:40:182750 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2751 sink_.WaitForEncodedFrame(2);
2752 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2753
Taylor Brandstetter49fcc102018-05-16 21:20:412754 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-16 06:40:182755 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512756 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412757 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 10:42:122758 // Give the encoder queue time to process the change in degradation preference
2759 // by waiting for an encoded frame.
2760 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2761 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 08:42:192762 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-16 06:40:182763
2764 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 15:27:512765 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 10:42:122766 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2767 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-16 06:40:182768 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2769
2770 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512771 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192772 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-16 06:40:182773
mflodmancc3d4422017-08-03 15:27:512774 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:182775}
2776
mflodmancc3d4422017-08-03 15:27:512777TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 08:47:312778 const int kWidth = 1280;
2779 const int kHeight = 720;
2780 const size_t kNumFrames = 10;
2781
Erik Språng4c6ca302019-04-08 13:14:012782 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072783 DataRate::BitsPerSec(kTargetBitrateBps),
2784 DataRate::BitsPerSec(kTargetBitrateBps),
2785 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 11:59:512786
asaperssond0de2952017-04-21 08:47:312787 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 15:58:542788 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 08:47:312789 video_source_.set_adaptation_enabled(true);
2790
2791 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2792 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2793
2794 int downscales = 0;
2795 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 08:42:192796 video_source_.IncomingCapturedFrame(
2797 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2798 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 08:47:312799
asaperssonfab67072017-04-04 12:51:492800 // Trigger scale down.
asaperssond0de2952017-04-21 08:47:312801 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 15:27:512802 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-03 06:53:042803 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 08:47:312804
2805 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2806 ++downscales;
2807
2808 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2809 EXPECT_EQ(downscales,
2810 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2811 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 11:59:512812 }
mflodmancc3d4422017-08-03 15:27:512813 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:312814}
2815
mflodmancc3d4422017-08-03 15:27:512816TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:312817 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2818 const int kWidth = 1280;
2819 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012820 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072821 DataRate::BitsPerSec(kTargetBitrateBps),
2822 DataRate::BitsPerSec(kTargetBitrateBps),
2823 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:312824
Taylor Brandstetter49fcc102018-05-16 21:20:412825 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:312826 AdaptingFrameForwarder source;
2827 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:512828 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412829 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:312830
Åsa Persson8c1bf952018-09-13 08:42:192831 int64_t timestamp_ms = kFrameIntervalMs;
2832 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522833 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192834 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:312835 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2836 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2837
2838 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512839 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:192840 timestamp_ms += kFrameIntervalMs;
2841 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2842 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:182843 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 08:47:312844 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2845 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2846
2847 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512848 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:192849 timestamp_ms += kFrameIntervalMs;
2850 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522851 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192852 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:312853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2854 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2855
2856 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512857 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:192858 timestamp_ms += kFrameIntervalMs;
2859 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2860 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:182861 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 08:47:312862 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2863 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2864
2865 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512866 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:192867 timestamp_ms += kFrameIntervalMs;
2868 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-16 06:40:182869 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192870 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:312871 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2872 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2873
mflodmancc3d4422017-08-03 15:27:512874 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:312875}
2876
mflodmancc3d4422017-08-03 15:27:512877TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-14 06:25:222878 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2879 const int kWidth = 1280;
2880 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012881 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072882 DataRate::BitsPerSec(kTargetBitrateBps),
2883 DataRate::BitsPerSec(kTargetBitrateBps),
2884 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:222885
Taylor Brandstetter49fcc102018-05-16 21:20:412886 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:222887 AdaptingFrameForwarder source;
2888 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:412889 video_stream_encoder_->SetSource(&source,
2890 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:222891
Åsa Persson8c1bf952018-09-13 08:42:192892 int64_t timestamp_ms = kFrameIntervalMs;
2893 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:222894 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192895 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222896 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2897 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2898
2899 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512900 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:192901 timestamp_ms += kFrameIntervalMs;
2902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2903 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:222904 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2905 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2906 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2907
2908 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512909 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192910 timestamp_ms += kFrameIntervalMs;
2911 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:222912 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192913 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222914 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2915 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2916
2917 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512918 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:192919 timestamp_ms += kFrameIntervalMs;
2920 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2921 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:222922 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2923 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2924 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2925
2926 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512927 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192928 timestamp_ms += kFrameIntervalMs;
2929 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:222930 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192931 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222932 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2933 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2934
mflodmancc3d4422017-08-03 15:27:512935 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222936}
2937
Sergey Silkin41c650b2019-10-14 11:12:192938TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
2939 fake_encoder_.SetResolutionBitrateLimits(
2940 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2941
2942 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072943 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2944 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2945 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2946 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:192947
2948 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2949 AdaptingFrameForwarder source;
2950 source.set_adaptation_enabled(true);
2951 video_stream_encoder_->SetSource(
2952 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2953
2954 // Insert 720p frame.
2955 int64_t timestamp_ms = kFrameIntervalMs;
2956 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2957 WaitForEncodedFrame(1280, 720);
2958
2959 // Reduce bitrate and trigger adapt down.
2960 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072961 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2962 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2963 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
2964 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:192965 video_stream_encoder_->TriggerQualityLow();
2966
2967 // Insert 720p frame. It should be downscaled and encoded.
2968 timestamp_ms += kFrameIntervalMs;
2969 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2970 WaitForEncodedFrame(960, 540);
2971
2972 // Trigger adapt up. Higher resolution should not be requested duo to lack
2973 // of bitrate.
2974 video_stream_encoder_->TriggerQualityHigh();
2975 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2976
2977 // Increase bitrate.
2978 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072979 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2980 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2981 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2982 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:192983
2984 // Trigger adapt up. Higher resolution should be requested.
2985 video_stream_encoder_->TriggerQualityHigh();
2986 VerifyFpsMaxResolutionMax(source.sink_wants());
2987
2988 video_stream_encoder_->Stop();
2989}
2990
2991TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
2992 fake_encoder_.SetResolutionBitrateLimits(
2993 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2994
2995 // Set bitrate equal to min bitrate of 540p.
2996 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:072997 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2998 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2999 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3000 0, 0);
Sergey Silkin41c650b2019-10-14 11:12:193001
3002 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3003 AdaptingFrameForwarder source;
3004 source.set_adaptation_enabled(true);
3005 video_stream_encoder_->SetSource(
3006 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3007
3008 // Insert 720p frame. It should be dropped and lower resolution should be
3009 // requested.
3010 int64_t timestamp_ms = kFrameIntervalMs;
3011 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3012 ExpectDroppedFrame();
3013 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3014
3015 // Insert 720p frame. It should be downscaled and encoded.
3016 timestamp_ms += kFrameIntervalMs;
3017 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3018 WaitForEncodedFrame(960, 540);
3019
3020 video_stream_encoder_->Stop();
3021}
3022
Åsa Perssonb67c44c2019-09-24 13:25:323023class BalancedDegradationTest : public VideoStreamEncoderTest {
3024 protected:
3025 void SetupTest() {
3026 // Reset encoder for field trials to take effect.
3027 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 13:13:043028 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 13:25:323029
3030 // Enable BALANCED preference.
3031 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 13:13:043032 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3033 }
3034
3035 void OnBitrateUpdated(int bitrate_bps) {
Ying Wang9b881ab2020-02-07 13:29:323036 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073037 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3038 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 13:25:323039 }
3040
Åsa Persson45b176f2019-09-30 09:19:053041 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 13:25:323042 timestamp_ms_ += kFrameIntervalMs;
3043 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 09:19:053044 }
3045
3046 void InsertFrameAndWaitForEncoded() {
3047 InsertFrame();
Åsa Perssonb67c44c2019-09-24 13:25:323048 sink_.WaitForEncodedFrame(timestamp_ms_);
3049 }
3050
3051 const int kWidth = 640; // pixels:640x360=230400
3052 const int kHeight = 360;
3053 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3054 int64_t timestamp_ms_ = 0;
3055 AdaptingFrameForwarder source_;
3056};
3057
3058TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3059 test::ScopedFieldTrials field_trials(
3060 "WebRTC-Video-BalancedDegradationSettings/"
3061 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3062 SetupTest();
3063
3064 // Force input frame rate.
3065 const int kInputFps = 24;
3066 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3067 stats.input_frame_rate = kInputFps;
3068 stats_proxy_->SetMockStats(stats);
3069
Åsa Persson45b176f2019-09-30 09:19:053070 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 13:25:323071 VerifyFpsMaxResolutionMax(source_.sink_wants());
3072
3073 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
3074 // Fps diff (input-requested:0) < threshold, expect AdaptDown to return false.
3075 video_stream_encoder_->TriggerQualityLowExpectFalse();
3076 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3077
3078 video_stream_encoder_->Stop();
3079}
3080
3081TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3082 test::ScopedFieldTrials field_trials(
3083 "WebRTC-Video-BalancedDegradationSettings/"
3084 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3085 SetupTest();
3086
3087 // Force input frame rate.
3088 const int kInputFps = 25;
3089 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3090 stats.input_frame_rate = kInputFps;
3091 stats_proxy_->SetMockStats(stats);
3092
Åsa Persson45b176f2019-09-30 09:19:053093 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 13:25:323094 VerifyFpsMaxResolutionMax(source_.sink_wants());
3095
3096 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
3097 // Fps diff (input-requested:1) == threshold, expect AdaptDown to return true.
3098 video_stream_encoder_->TriggerQualityLow();
3099 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3100
3101 video_stream_encoder_->Stop();
3102}
3103
3104TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3105 test::ScopedFieldTrials field_trials(
3106 "WebRTC-Video-BalancedDegradationSettings/"
3107 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3108 SetupTest();
3109
3110 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3111
Åsa Persson45b176f2019-09-30 09:19:053112 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 13:25:323113 VerifyFpsMaxResolutionMax(source_.sink_wants());
3114
3115 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3116 video_stream_encoder_->TriggerQualityLow();
3117 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
3118
3119 video_stream_encoder_->Stop();
3120}
3121
Åsa Perssonccfb3402019-09-25 13:13:043122TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:323123 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 15:26:393124 "WebRTC-Video-BalancedDegradationSettings/"
3125 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 13:13:043126 SetupTest();
Åsa Persson1b247f12019-08-14 15:26:393127
Åsa Persson1b247f12019-08-14 15:26:393128 const int kMinBitrateBps = 425000;
3129 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 13:13:043130 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 15:26:393131
Åsa Persson45b176f2019-09-30 09:19:053132 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043133 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 15:26:393134 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3135
3136 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3137 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053138 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043139 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 15:26:393140 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3141
3142 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3143 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053144 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043145 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 15:26:393146 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3147
Åsa Persson30ab0152019-08-27 10:22:333148 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3149 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053150 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043151 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3152 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 10:22:333153 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3154
3155 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 15:26:393156 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053157 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:333158 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 15:26:393159
Åsa Persson30ab0152019-08-27 10:22:333160 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043161 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 15:26:393162 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053163 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043164 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 10:22:333165 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3166
3167 video_stream_encoder_->Stop();
3168}
3169
Åsa Perssonccfb3402019-09-25 13:13:043170TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 09:19:053171 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3172 test::ScopedFieldTrials field_trials(
3173 "WebRTC-Video-BalancedDegradationSettings/"
3174 "pixels:57600|129600|230400,fps:7|24|24/");
3175 SetupTest();
3176 OnBitrateUpdated(kLowTargetBitrateBps);
3177
3178 VerifyNoLimitation(source_.sink_wants());
3179
3180 // Insert frame, expect scaled down:
3181 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3182 InsertFrame();
3183 EXPECT_FALSE(WaitForFrame(1000));
3184 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3185 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3186
3187 // Insert frame, expect scaled down:
3188 // resolution (320x180@24fps).
3189 InsertFrame();
3190 EXPECT_FALSE(WaitForFrame(1000));
3191 EXPECT_LT(source_.sink_wants().max_pixel_count,
3192 source_.last_wants().max_pixel_count);
3193 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3194
3195 // Frame should not be dropped (min pixels per frame reached).
3196 InsertFrameAndWaitForEncoded();
3197
3198 video_stream_encoder_->Stop();
3199}
3200
3201TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:333202 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:323203 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:333204 "WebRTC-Video-BalancedDegradationSettings/"
3205 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:043206 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:333207
Åsa Persson30ab0152019-08-27 10:22:333208 const int kResolutionMinBitrateBps = 435000;
3209 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 13:13:043210 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333211
Åsa Persson45b176f2019-09-30 09:19:053212 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043213 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 10:22:333214 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3215
3216 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3217 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053218 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043219 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 10:22:333220 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3221
3222 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3223 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053224 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043225 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333226 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3227
3228 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3229 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053230 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043231 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 15:26:393232 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3233
Åsa Persson30ab0152019-08-27 10:22:333234 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3235 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053236 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043237 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333238 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3239
3240 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3241 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053242 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:333243 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3244
3245 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043246 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333247 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053248 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043249 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333250 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3251
3252 video_stream_encoder_->Stop();
3253}
3254
Åsa Perssonccfb3402019-09-25 13:13:043255TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:333256 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:323257 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:333258 "WebRTC-Video-BalancedDegradationSettings/"
3259 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:043260 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:333261
Åsa Persson30ab0152019-08-27 10:22:333262 const int kMinBitrateBps = 425000;
3263 const int kTooLowMinBitrateBps = 424000;
3264 const int kResolutionMinBitrateBps = 435000;
3265 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 13:13:043266 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333267
Åsa Persson45b176f2019-09-30 09:19:053268 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043269 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 10:22:333270 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3271
3272 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3273 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053274 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043275 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 10:22:333276 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3277
3278 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3279 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053280 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043281 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333282 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3283
3284 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3285 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053286 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043287 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333288 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3289
3290 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3291 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053292 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:333293 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3294
3295 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043296 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333297 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053298 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043299 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333300 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3301
3302 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043303 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333304 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053305 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:333306 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3307
3308 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043309 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333310 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053311 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043312 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333313 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3314
Åsa Persson1b247f12019-08-14 15:26:393315 video_stream_encoder_->Stop();
3316}
3317
mflodmancc3d4422017-08-03 15:27:513318TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:313319 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3320 const int kWidth = 1280;
3321 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:013322 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073323 DataRate::BitsPerSec(kTargetBitrateBps),
3324 DataRate::BitsPerSec(kTargetBitrateBps),
3325 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:313326
Taylor Brandstetter49fcc102018-05-16 21:20:413327 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:313328 AdaptingFrameForwarder source;
3329 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513330 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413331 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:313332
Åsa Persson8c1bf952018-09-13 08:42:193333 int64_t timestamp_ms = kFrameIntervalMs;
3334 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523335 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:193336 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:313337 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3338 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3339 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3340 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3341
3342 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 15:27:513343 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193344 timestamp_ms += kFrameIntervalMs;
3345 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3346 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183347 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 08:47:313348 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3349 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3350 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3351 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3352
3353 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 15:27:513354 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193355 timestamp_ms += kFrameIntervalMs;
3356 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3357 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183358 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 08:47:313359 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3360 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3361 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3362 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3363
Jonathan Yubc771b72017-12-09 01:04:293364 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 15:27:513365 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193366 timestamp_ms += kFrameIntervalMs;
3367 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3368 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-09 01:04:293369 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 08:47:313370 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3371 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:293372 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:313373 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3374
Jonathan Yubc771b72017-12-09 01:04:293375 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 15:27:513376 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193377 timestamp_ms += kFrameIntervalMs;
3378 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3379 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183380 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-09 01:04:293381 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 08:47:313382 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3383 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3384 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3385 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3386
Jonathan Yubc771b72017-12-09 01:04:293387 // Trigger quality adapt down, expect no change (min resolution reached).
3388 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193389 timestamp_ms += kFrameIntervalMs;
3390 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3391 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-09 01:04:293392 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3393 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3394 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3395 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3396 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3397
3398 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 15:27:513399 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:193400 timestamp_ms += kFrameIntervalMs;
3401 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3402 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183403 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-09 01:04:293404 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3405 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3406 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3407 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3408
3409 // Trigger cpu adapt up, expect upscaled resolution (640x360).
3410 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:193411 timestamp_ms += kFrameIntervalMs;
3412 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3413 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-09 01:04:293414 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3415 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3416 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3417 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3418 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3419
3420 // Trigger cpu adapt up, expect upscaled resolution (960x540).
3421 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:193422 timestamp_ms += kFrameIntervalMs;
3423 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3424 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-09 01:04:293425 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 08:47:313426 last_wants = source.sink_wants();
3427 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3428 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:293429 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:313430 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3431
3432 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 15:27:513433 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:193434 timestamp_ms += kFrameIntervalMs;
3435 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3436 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183437 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 08:47:313438 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3439 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:293440 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:313441 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3442
3443 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 15:27:513444 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193445 timestamp_ms += kFrameIntervalMs;
3446 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523447 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-16 06:40:183448 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 08:42:193449 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:313450 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3451 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:293452 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:313453 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 11:59:513454
mflodmancc3d4422017-08-03 15:27:513455 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 11:59:513456}
3457
mflodmancc3d4422017-08-03 15:27:513458TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 12:51:493459 const int kWidth = 640;
3460 const int kHeight = 360;
perkj803d97f2016-11-01 18:45:463461
Erik Språng4c6ca302019-04-08 13:14:013462 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073463 DataRate::BitsPerSec(kTargetBitrateBps),
3464 DataRate::BitsPerSec(kTargetBitrateBps),
3465 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:523466
perkj803d97f2016-11-01 18:45:463467 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:493468 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523469 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 18:45:463470 }
3471
mflodmancc3d4422017-08-03 15:27:513472 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:463473 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:493474 video_source_.IncomingCapturedFrame(CreateFrame(
3475 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523476 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 18:45:463477 }
3478
mflodmancc3d4422017-08-03 15:27:513479 video_stream_encoder_->Stop();
3480 video_stream_encoder_.reset();
perkj803d97f2016-11-01 18:45:463481 stats_proxy_.reset();
sprangf8ee65e2017-02-28 16:49:333482
Ying Wangef3998f2019-12-09 12:06:533483 EXPECT_METRIC_EQ(
3484 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3485 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 18:45:463486 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3487}
3488
mflodmancc3d4422017-08-03 15:27:513489TEST_F(VideoStreamEncoderTest,
3490 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 13:14:013491 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073492 DataRate::BitsPerSec(kTargetBitrateBps),
3493 DataRate::BitsPerSec(kTargetBitrateBps),
3494 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 09:01:063495 const int kWidth = 640;
3496 const int kHeight = 360;
3497
Taylor Brandstetter49fcc102018-05-16 21:20:413498 video_stream_encoder_->SetSource(&video_source_,
3499 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 09:01:063500
3501 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3502 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523503 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 09:01:063504 }
3505
mflodmancc3d4422017-08-03 15:27:513506 video_stream_encoder_->Stop();
3507 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 09:01:063508 stats_proxy_.reset();
3509
3510 EXPECT_EQ(0,
3511 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3512}
3513
mflodmancc3d4422017-08-03 15:27:513514TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 14:06:523515 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 12:09:313516 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 14:24:023517
3518 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 10:32:223519 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 14:36:493520 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 13:16:283521 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3522 kDefaultFps));
sprang57c2fff2017-01-16 14:24:023523
sprang57c2fff2017-01-16 14:24:023524 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 10:11:103525 .Times(1);
Florent Castellia8336d32019-09-09 11:36:553526 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073527 DataRate::BitsPerSec(kLowTargetBitrateBps),
3528 DataRate::BitsPerSec(kLowTargetBitrateBps),
3529 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 14:24:023530
sprang57c2fff2017-01-16 14:24:023531 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 20:19:533532 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3533 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 13:53:113534 VideoBitrateAllocation bitrate_allocation =
3535 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 20:19:533536 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 13:53:113537 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 10:11:203538 // TODO(srte): The use of millisecs here looks like an error, but the tests
3539 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 10:16:003540 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:023541
3542 // Not called on second frame.
3543 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3544 .Times(0);
3545 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 20:19:533546 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3547 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 10:16:003548 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:023549
3550 // Called after a process interval.
sprang57c2fff2017-01-16 14:24:023551 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3552 .Times(1);
Erik Språngd7329ca2019-02-21 20:19:533553 const int64_t start_time_ms = rtc::TimeMillis();
3554 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3555 video_source_.IncomingCapturedFrame(
3556 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3557 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 10:16:003558 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 20:19:533559 }
3560
3561 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 13:53:113562 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 14:24:023563
mflodmancc3d4422017-08-03 15:27:513564 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 14:24:023565}
3566
Åsa Perssonc29cb2c2019-03-25 11:06:593567TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3568 // 2 TLs configured, temporal layers supported by encoder.
3569 const int kNumTemporalLayers = 2;
3570 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3571 fake_encoder_.SetTemporalLayersSupported(0, true);
3572
3573 // Bitrate allocated across temporal layers.
3574 const int kTl0Bps = kTargetBitrateBps *
3575 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:463576 kNumTemporalLayers, /*temporal_id*/ 0,
3577 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:593578 const int kTl1Bps = kTargetBitrateBps *
3579 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:463580 kNumTemporalLayers, /*temporal_id*/ 1,
3581 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:593582 VideoBitrateAllocation expected_bitrate;
3583 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3584 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3585
3586 VerifyAllocatedBitrate(expected_bitrate);
3587 video_stream_encoder_->Stop();
3588}
3589
3590TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3591 // 2 TLs configured, temporal layers not supported by encoder.
3592 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3593 fake_encoder_.SetTemporalLayersSupported(0, false);
3594
3595 // Temporal layers not supported by the encoder.
3596 // Total bitrate should be at ti:0.
3597 VideoBitrateAllocation expected_bitrate;
3598 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3599
3600 VerifyAllocatedBitrate(expected_bitrate);
3601 video_stream_encoder_->Stop();
3602}
3603
3604TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3605 // 2 TLs configured, temporal layers only supported for first stream.
3606 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3607 fake_encoder_.SetTemporalLayersSupported(0, true);
3608 fake_encoder_.SetTemporalLayersSupported(1, false);
3609
3610 const int kS0Bps = 150000;
3611 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:463612 kS0Bps *
3613 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3614 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:593615 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:463616 kS0Bps *
3617 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3618 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:593619 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3620 // Temporal layers not supported by si:1.
3621 VideoBitrateAllocation expected_bitrate;
3622 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3623 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3624 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3625
3626 VerifyAllocatedBitrate(expected_bitrate);
3627 video_stream_encoder_->Stop();
3628}
3629
Niels Möller7dc26b72017-12-06 09:27:483630TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3631 const int kFrameWidth = 1280;
3632 const int kFrameHeight = 720;
3633 const int kFramerate = 24;
3634
Erik Språng4c6ca302019-04-08 13:14:013635 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073636 DataRate::BitsPerSec(kTargetBitrateBps),
3637 DataRate::BitsPerSec(kTargetBitrateBps),
3638 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:483639 test::FrameForwarder source;
3640 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413641 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:483642
3643 // Insert a single frame, triggering initial configuration.
3644 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3645 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3646
3647 EXPECT_EQ(
3648 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3649 kDefaultFramerate);
3650
3651 // Trigger reconfigure encoder (without resetting the entire instance).
3652 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:513653 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 09:27:483654 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3655 video_encoder_config.number_of_streams = 1;
3656 video_encoder_config.video_stream_factory =
3657 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3658 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473659 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:483660 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3661
3662 // Detector should be updated with fps limit from codec config.
3663 EXPECT_EQ(
3664 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3665 kFramerate);
3666
3667 // Trigger overuse, max framerate should be reduced.
3668 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3669 stats.input_frame_rate = kFramerate;
3670 stats_proxy_->SetMockStats(stats);
3671 video_stream_encoder_->TriggerCpuOveruse();
3672 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3673 int adapted_framerate =
3674 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3675 EXPECT_LT(adapted_framerate, kFramerate);
3676
3677 // Trigger underuse, max framerate should go back to codec configured fps.
3678 // Set extra low fps, to make sure it's actually reset, not just incremented.
3679 stats = stats_proxy_->GetStats();
3680 stats.input_frame_rate = adapted_framerate / 2;
3681 stats_proxy_->SetMockStats(stats);
3682 video_stream_encoder_->TriggerCpuNormalUsage();
3683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3684 EXPECT_EQ(
3685 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3686 kFramerate);
3687
3688 video_stream_encoder_->Stop();
3689}
3690
3691TEST_F(VideoStreamEncoderTest,
3692 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3693 const int kFrameWidth = 1280;
3694 const int kFrameHeight = 720;
3695 const int kLowFramerate = 15;
3696 const int kHighFramerate = 25;
3697
Erik Språng4c6ca302019-04-08 13:14:013698 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073699 DataRate::BitsPerSec(kTargetBitrateBps),
3700 DataRate::BitsPerSec(kTargetBitrateBps),
3701 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 09:27:483702 test::FrameForwarder source;
3703 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413704 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:483705
3706 // Trigger initial configuration.
3707 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:513708 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 09:27:483709 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3710 video_encoder_config.number_of_streams = 1;
3711 video_encoder_config.video_stream_factory =
3712 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3713 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3714 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473715 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:483716 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3717
3718 EXPECT_EQ(
3719 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3720 kLowFramerate);
3721
3722 // Trigger overuse, max framerate should be reduced.
3723 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3724 stats.input_frame_rate = kLowFramerate;
3725 stats_proxy_->SetMockStats(stats);
3726 video_stream_encoder_->TriggerCpuOveruse();
3727 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3728 int adapted_framerate =
3729 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3730 EXPECT_LT(adapted_framerate, kLowFramerate);
3731
3732 // Reconfigure the encoder with a new (higher max framerate), max fps should
3733 // still respect the adaptation.
3734 video_encoder_config.video_stream_factory =
3735 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3736 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3737 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473738 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:483739 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3740
3741 EXPECT_EQ(
3742 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3743 adapted_framerate);
3744
3745 // Trigger underuse, max framerate should go back to codec configured fps.
3746 stats = stats_proxy_->GetStats();
3747 stats.input_frame_rate = adapted_framerate;
3748 stats_proxy_->SetMockStats(stats);
3749 video_stream_encoder_->TriggerCpuNormalUsage();
3750 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3751 EXPECT_EQ(
3752 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3753 kHighFramerate);
3754
3755 video_stream_encoder_->Stop();
3756}
3757
mflodmancc3d4422017-08-03 15:27:513758TEST_F(VideoStreamEncoderTest,
3759 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 11:21:073760 const int kFrameWidth = 1280;
3761 const int kFrameHeight = 720;
3762 const int kFramerate = 24;
3763
Erik Språng4c6ca302019-04-08 13:14:013764 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073765 DataRate::BitsPerSec(kTargetBitrateBps),
3766 DataRate::BitsPerSec(kTargetBitrateBps),
3767 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 11:21:073768 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513769 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413770 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 11:21:073771
3772 // Trigger initial configuration.
3773 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:513774 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 11:21:073775 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3776 video_encoder_config.number_of_streams = 1;
3777 video_encoder_config.video_stream_factory =
3778 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3779 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 15:27:513780 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473781 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:513782 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 11:21:073783
Niels Möller7dc26b72017-12-06 09:27:483784 EXPECT_EQ(
3785 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3786 kFramerate);
sprangfda496a2017-06-15 11:21:073787
3788 // Trigger overuse, max framerate should be reduced.
3789 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3790 stats.input_frame_rate = kFramerate;
3791 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 15:27:513792 video_stream_encoder_->TriggerCpuOveruse();
3793 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:483794 int adapted_framerate =
3795 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 11:21:073796 EXPECT_LT(adapted_framerate, kFramerate);
3797
3798 // Change degradation preference to not enable framerate scaling. Target
3799 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 15:27:513800 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413801 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 15:27:513802 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:483803 EXPECT_EQ(
3804 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3805 kFramerate);
sprangfda496a2017-06-15 11:21:073806
mflodmancc3d4422017-08-03 15:27:513807 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 11:21:073808}
3809
mflodmancc3d4422017-08-03 15:27:513810TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:493811 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 14:37:333812 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073813 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3814 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3815 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:493816 const int kWidth = 640;
3817 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:223818
asaperssonfab67072017-04-04 12:51:493819 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:223820
3821 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:523822 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:223823
3824 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 07:57:583825 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:223826
sprangc5d62e22017-04-03 06:53:043827 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 15:02:223828
asaperssonfab67072017-04-04 12:51:493829 // Next frame is scaled.
kthelgason2bc68642017-02-07 15:02:223830 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493831 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 15:02:223832
3833 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:523834 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:223835
sprangc5d62e22017-04-03 06:53:043836 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 15:02:223837
mflodmancc3d4422017-08-03 15:27:513838 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:223839}
3840
mflodmancc3d4422017-08-03 15:27:513841TEST_F(VideoStreamEncoderTest,
3842 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:493843 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 14:37:333844 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073845 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3846 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3847 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 12:51:493848 const int kWidth = 640;
3849 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:223850
3851 // We expect the n initial frames to get dropped.
3852 int i;
3853 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 12:51:493854 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523855 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:223856 }
3857 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 12:51:493858 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523859 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 15:02:223860
3861 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 12:51:493862 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:223863
mflodmancc3d4422017-08-03 15:27:513864 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:223865}
3866
mflodmancc3d4422017-08-03 15:27:513867TEST_F(VideoStreamEncoderTest,
3868 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 12:51:493869 const int kWidth = 640;
3870 const int kHeight = 360;
Florent Castellia8336d32019-09-09 11:36:553871 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073872 DataRate::BitsPerSec(kLowTargetBitrateBps),
3873 DataRate::BitsPerSec(kLowTargetBitrateBps),
3874 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 15:02:223875
3876 // Set degradation preference.
mflodmancc3d4422017-08-03 15:27:513877 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413878 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 15:02:223879
asaperssonfab67072017-04-04 12:51:493880 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:223881 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:523882 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 15:02:223883
mflodmancc3d4422017-08-03 15:27:513884 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:223885}
3886
mflodmancc3d4422017-08-03 15:27:513887TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 12:51:493888 const int kWidth = 640;
3889 const int kHeight = 360;
kthelgasonad9010c2017-02-14 08:46:513890 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:133891
3892 VideoEncoderConfig video_encoder_config;
3893 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3894 // Make format different, to force recreation of encoder.
3895 video_encoder_config.video_format.parameters["foo"] = "foo";
3896 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473897 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 11:36:553898 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073899 DataRate::BitsPerSec(kLowTargetBitrateBps),
3900 DataRate::BitsPerSec(kLowTargetBitrateBps),
3901 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-16 06:40:183902
kthelgasonb83797b2017-02-14 19:57:253903 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 21:20:413904 video_stream_encoder_->SetSource(&video_source_,
3905 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 08:46:513906
asaperssonfab67072017-04-04 12:51:493907 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 08:46:513908 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:523909 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 08:46:513910
mflodmancc3d4422017-08-03 15:27:513911 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 08:46:513912 fake_encoder_.SetQualityScaling(true);
3913}
3914
Åsa Persson139f4dc2019-08-02 07:29:583915TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3916 webrtc::test::ScopedFieldTrials field_trials(
3917 "WebRTC-Video-QualityScalerSettings/"
3918 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3919 // Reset encoder for field trials to take effect.
3920 ConfigureEncoder(video_encoder_config_.Copy());
3921 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3922 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3923 const int kWidth = 640;
3924 const int kHeight = 360;
3925
3926 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073927 DataRate::BitsPerSec(kTargetBitrateBps),
3928 DataRate::BitsPerSec(kTargetBitrateBps),
3929 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:583930 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3931 // Frame should not be dropped.
3932 WaitForEncodedFrame(1);
3933
3934 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073935 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3936 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3937 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:583938 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3939 // Frame should not be dropped.
3940 WaitForEncodedFrame(2);
3941
3942 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:073943 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3944 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3945 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:583946 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3947 // Expect to drop this frame, the wait should time out.
3948 ExpectDroppedFrame();
3949
3950 // Expect the sink_wants to specify a scaled frame.
3951 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3952 video_stream_encoder_->Stop();
3953}
3954
Åsa Perssone644a032019-11-08 14:56:003955TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
3956 webrtc::test::ScopedFieldTrials field_trials(
3957 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
3958
3959 // Reset encoder for field trials to take effect.
3960 VideoEncoderConfig config = video_encoder_config_.Copy();
3961 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 09:34:323962 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 14:56:003963 ConfigureEncoder(std::move(config));
3964 fake_encoder_.SetQp(kQpLow);
3965
3966 // Enable MAINTAIN_FRAMERATE preference.
3967 AdaptingFrameForwarder source;
3968 source.set_adaptation_enabled(true);
3969 video_stream_encoder_->SetSource(&source,
3970 DegradationPreference::MAINTAIN_FRAMERATE);
3971
3972 // Start at low bitrate.
3973 const int kLowBitrateBps = 200000;
Danil Chapovalovcad3e0e2020-02-17 17:46:073974 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(kLowBitrateBps),
3975 DataRate::BitsPerSec(kLowBitrateBps),
3976 DataRate::BitsPerSec(kLowBitrateBps),
3977 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:003978
3979 // Expect first frame to be dropped and resolution to be limited.
3980 const int kWidth = 1280;
3981 const int kHeight = 720;
3982 const int64_t kFrameIntervalMs = 100;
3983 int64_t timestamp_ms = kFrameIntervalMs;
3984 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3985 ExpectDroppedFrame();
3986 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3987
3988 // Increase bitrate to encoder max.
Evan Shrubsoledff79252020-04-16 09:34:323989 video_stream_encoder_->OnBitrateUpdated(max_bitrate, max_bitrate, max_bitrate,
3990 0, 0, 0);
Åsa Perssone644a032019-11-08 14:56:003991
3992 // Insert frames and advance |min_duration_ms|.
3993 for (size_t i = 1; i <= 10; i++) {
3994 timestamp_ms += kFrameIntervalMs;
3995 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3996 WaitForEncodedFrame(timestamp_ms);
3997 }
3998 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3999 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4000
Danil Chapovalov0c626af2020-02-10 10:16:004001 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 14:56:004002
4003 // Insert frame should trigger high BW and release quality limitation.
4004 timestamp_ms += kFrameIntervalMs;
4005 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4006 WaitForEncodedFrame(timestamp_ms);
4007 VerifyFpsMaxResolutionMax(source.sink_wants());
4008
4009 // Frame should not be adapted.
4010 timestamp_ms += kFrameIntervalMs;
4011 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4012 WaitForEncodedFrame(kWidth, kHeight);
4013 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4014
4015 video_stream_encoder_->Stop();
4016}
4017
mflodmancc3d4422017-08-03 15:27:514018TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:314019 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4020 const int kTooSmallWidth = 10;
4021 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 13:14:014022 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074023 DataRate::BitsPerSec(kTargetBitrateBps),
4024 DataRate::BitsPerSec(kTargetBitrateBps),
4025 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 08:47:314026
Taylor Brandstetter49fcc102018-05-16 21:20:414027 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:314028 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:514029 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414030 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:314031 VerifyNoLimitation(source.sink_wants());
4032 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4033
4034 // Trigger adapt down, too small frame, expect no change.
4035 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:524036 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:514037 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:194038 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:314039 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4040 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4041
mflodmancc3d4422017-08-03 15:27:514042 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:314043}
4044
mflodmancc3d4422017-08-03 15:27:514045TEST_F(VideoStreamEncoderTest,
4046 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:224047 const int kTooSmallWidth = 10;
4048 const int kTooSmallHeight = 10;
4049 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 13:14:014050 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074051 DataRate::BitsPerSec(kTargetBitrateBps),
4052 DataRate::BitsPerSec(kTargetBitrateBps),
4053 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:224054
Taylor Brandstetter49fcc102018-05-16 21:20:414055 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:224056 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:414057 video_stream_encoder_->SetSource(&source,
4058 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:224059 VerifyNoLimitation(source.sink_wants());
4060 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4061 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4062
4063 // Trigger adapt down, expect limited framerate.
4064 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:524065 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:514066 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224067 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4068 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4069 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4070 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4071
4072 // Trigger adapt down, too small frame, expect no change.
4073 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:524074 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:514075 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224076 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4077 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4078 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4079 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4080
mflodmancc3d4422017-08-03 15:27:514081 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224082}
4083
mflodmancc3d4422017-08-03 15:27:514084TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 08:12:524085 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 13:14:014086 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074087 DataRate::BitsPerSec(kTargetBitrateBps),
4088 DataRate::BitsPerSec(kTargetBitrateBps),
4089 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 07:51:474090 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 08:12:524091 const int kFrameWidth = 1280;
4092 const int kFrameHeight = 720;
4093 video_source_.IncomingCapturedFrame(
4094 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524095 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 15:27:514096 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:524097}
4098
sprangb1ca0732017-02-01 16:38:124099// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 15:27:514100TEST_F(VideoStreamEncoderTest,
4101 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 13:14:014102 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074103 DataRate::BitsPerSec(kTargetBitrateBps),
4104 DataRate::BitsPerSec(kTargetBitrateBps),
4105 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 16:38:124106
4107 const int kFrameWidth = 1280;
4108 const int kFrameHeight = 720;
4109 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 15:27:514110 // requested by
4111 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 16:38:124112 video_source_.set_adaptation_enabled(true);
4113
4114 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:194115 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524116 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:124117
4118 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:514119 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 16:38:124120 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:194121 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524122 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 16:38:124123
asaperssonfab67072017-04-04 12:51:494124 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 15:27:514125 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 16:38:124126 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:194127 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524128 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:124129
mflodmancc3d4422017-08-03 15:27:514130 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 16:38:124131}
sprangfe627f32017-03-29 15:24:594132
mflodmancc3d4422017-08-03 15:27:514133TEST_F(VideoStreamEncoderTest,
4134 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-03 06:53:044135 const int kFrameWidth = 1280;
4136 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:044137
Erik Språng4c6ca302019-04-08 13:14:014138 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074139 DataRate::BitsPerSec(kTargetBitrateBps),
4140 DataRate::BitsPerSec(kTargetBitrateBps),
4141 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:514142 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414143 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:044144 video_source_.set_adaptation_enabled(true);
4145
sprang4847ae62017-06-27 14:06:524146 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-03 06:53:044147
4148 video_source_.IncomingCapturedFrame(
4149 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524150 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:044151
4152 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 15:27:514153 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:044154
4155 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 14:06:524156 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:044157 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:044158 video_source_.IncomingCapturedFrame(
4159 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524160 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:044161 }
4162
4163 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 15:27:514164 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:044165 int num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:524166 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:044167 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:044168 video_source_.IncomingCapturedFrame(
4169 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524170 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:044171 ++num_frames_dropped;
4172 } else {
Åsa Perssonc74d8da2017-12-04 13:13:564173 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:044174 }
4175 }
4176
sprang4847ae62017-06-27 14:06:524177 // Add some slack to account for frames dropped by the frame dropper.
4178 const int kErrorMargin = 1;
4179 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:044180 kErrorMargin);
4181
4182 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 15:27:514183 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:044184 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 08:42:194185 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:044186 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:044187 video_source_.IncomingCapturedFrame(
4188 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524189 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:044190 ++num_frames_dropped;
4191 } else {
Åsa Perssonc74d8da2017-12-04 13:13:564192 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:044193 }
4194 }
sprang4847ae62017-06-27 14:06:524195 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-03 06:53:044196 kErrorMargin);
4197
4198 // Go back up one step.
mflodmancc3d4422017-08-03 15:27:514199 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-03 06:53:044200 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:524201 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:044202 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:044203 video_source_.IncomingCapturedFrame(
4204 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524205 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:044206 ++num_frames_dropped;
4207 } else {
Åsa Perssonc74d8da2017-12-04 13:13:564208 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:044209 }
4210 }
sprang4847ae62017-06-27 14:06:524211 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:044212 kErrorMargin);
4213
4214 // Go back up to original mode.
mflodmancc3d4422017-08-03 15:27:514215 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-03 06:53:044216 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:524217 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:044218 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:044219 video_source_.IncomingCapturedFrame(
4220 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524221 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:044222 ++num_frames_dropped;
4223 } else {
Åsa Perssonc74d8da2017-12-04 13:13:564224 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:044225 }
4226 }
4227 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4228
mflodmancc3d4422017-08-03 15:27:514229 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:044230}
4231
mflodmancc3d4422017-08-03 15:27:514232TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-03 06:53:044233 const int kFramerateFps = 5;
4234 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-03 06:53:044235 const int kFrameWidth = 1280;
4236 const int kFrameHeight = 720;
4237
sprang4847ae62017-06-27 14:06:524238 // Reconfigure encoder with two temporal layers and screensharing, which will
4239 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 07:51:474240 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 14:06:524241
Erik Språng4c6ca302019-04-08 13:14:014242 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074243 DataRate::BitsPerSec(kTargetBitrateBps),
4244 DataRate::BitsPerSec(kTargetBitrateBps),
4245 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:514246 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414247 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:044248 video_source_.set_adaptation_enabled(true);
4249
sprang4847ae62017-06-27 14:06:524250 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-03 06:53:044251
4252 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-09 01:04:294253 rtc::VideoSinkWants last_wants;
4254 do {
4255 last_wants = video_source_.sink_wants();
4256
sprangc5d62e22017-04-03 06:53:044257 // Insert frames to get a new fps estimate...
4258 for (int j = 0; j < kFramerateFps; ++j) {
4259 video_source_.IncomingCapturedFrame(
4260 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-09 01:04:294261 if (video_source_.last_sent_width()) {
4262 sink_.WaitForEncodedFrame(timestamp_ms);
4263 }
sprangc5d62e22017-04-03 06:53:044264 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 10:16:004265 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-03 06:53:044266 }
4267 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 15:27:514268 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:294269 } while (video_source_.sink_wants().max_framerate_fps <
4270 last_wants.max_framerate_fps);
sprangc5d62e22017-04-03 06:53:044271
Jonathan Yubc771b72017-12-09 01:04:294272 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-14 06:25:224273
mflodmancc3d4422017-08-03 15:27:514274 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:044275}
asaperssonf7e294d2017-06-14 06:25:224276
mflodmancc3d4422017-08-03 15:27:514277TEST_F(VideoStreamEncoderTest,
4278 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:224279 const int kWidth = 1280;
4280 const int kHeight = 720;
4281 const int64_t kFrameIntervalMs = 150;
4282 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 13:14:014283 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074284 DataRate::BitsPerSec(kTargetBitrateBps),
4285 DataRate::BitsPerSec(kTargetBitrateBps),
4286 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:224287
Taylor Brandstetter49fcc102018-05-16 21:20:414288 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:224289 AdaptingFrameForwarder source;
4290 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:414291 video_stream_encoder_->SetSource(&source,
4292 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:224293 timestamp_ms += kFrameIntervalMs;
4294 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524295 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:194296 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4299 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4300
4301 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:514302 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224303 timestamp_ms += kFrameIntervalMs;
4304 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524305 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224306 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4307 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4308 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4309 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4310
4311 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514312 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224313 timestamp_ms += kFrameIntervalMs;
4314 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524315 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224316 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4317 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4318 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4319 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4320
4321 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514322 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224323 timestamp_ms += kFrameIntervalMs;
4324 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524325 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224326 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4327 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4329 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4330
4331 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:514332 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224333 timestamp_ms += kFrameIntervalMs;
4334 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524335 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224336 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4337 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4338 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4339 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4340
4341 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:514342 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224343 timestamp_ms += kFrameIntervalMs;
4344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524345 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224346 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4347 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4348 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4349 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4350
4351 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:514352 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224353 timestamp_ms += kFrameIntervalMs;
4354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524355 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224356 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4357 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4358 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4359 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4360
4361 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 15:27:514362 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224363 timestamp_ms += kFrameIntervalMs;
4364 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524365 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224366 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4367 rtc::VideoSinkWants last_wants = source.sink_wants();
4368 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4369 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4370 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4371
4372 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 15:27:514373 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224374 timestamp_ms += kFrameIntervalMs;
4375 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524376 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224377 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4378 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4380 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4381
4382 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:514383 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224384 timestamp_ms += kFrameIntervalMs;
4385 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524386 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224387 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4388 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4390 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4391
4392 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:514393 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224394 timestamp_ms += kFrameIntervalMs;
4395 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524396 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224397 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4398 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4399 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4400 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4401
4402 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:514403 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224404 timestamp_ms += kFrameIntervalMs;
4405 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524406 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224407 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4408 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4409 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4410 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4411
4412 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514413 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224414 timestamp_ms += kFrameIntervalMs;
4415 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524416 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224417 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4418 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4419 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4420 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4421
4422 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514423 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224424 timestamp_ms += kFrameIntervalMs;
4425 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524426 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224427 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4428 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4429 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4430 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4431
4432 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:514433 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224434 timestamp_ms += kFrameIntervalMs;
4435 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524436 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224437 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4439 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4440 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4441
Åsa Persson30ab0152019-08-27 10:22:334442 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:514443 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224444 timestamp_ms += kFrameIntervalMs;
4445 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524446 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-14 06:25:224447 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 08:42:194448 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224449 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4450 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4451 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4452
4453 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:514454 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194455 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224456 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4457
mflodmancc3d4422017-08-03 15:27:514458 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224459}
4460
mflodmancc3d4422017-08-03 15:27:514461TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-14 06:25:224462 const int kWidth = 1280;
4463 const int kHeight = 720;
4464 const int64_t kFrameIntervalMs = 150;
4465 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 13:14:014466 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074467 DataRate::BitsPerSec(kTargetBitrateBps),
4468 DataRate::BitsPerSec(kTargetBitrateBps),
4469 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:224470
Taylor Brandstetter49fcc102018-05-16 21:20:414471 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:224472 AdaptingFrameForwarder source;
4473 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:414474 video_stream_encoder_->SetSource(&source,
4475 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:224476 timestamp_ms += kFrameIntervalMs;
4477 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524478 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:194479 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4481 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4482 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4483 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4484 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4485 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4486
4487 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:514488 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:224489 timestamp_ms += kFrameIntervalMs;
4490 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524491 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224492 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4493 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4494 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4495 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4496 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4497 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4498 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4499
4500 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514501 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:224502 timestamp_ms += kFrameIntervalMs;
4503 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524504 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224505 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4506 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4507 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4508 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4509 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4510 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4511 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4512
4513 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514514 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224515 timestamp_ms += kFrameIntervalMs;
4516 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524517 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224518 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4519 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4520 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4521 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4522 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4523 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4524 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4525
4526 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514527 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-14 06:25:224528 timestamp_ms += kFrameIntervalMs;
4529 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524530 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224531 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4532 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4533 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4534 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4535 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4536 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4537 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4538
4539 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:514540 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224541 timestamp_ms += kFrameIntervalMs;
4542 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524543 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224544 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4545 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4546 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4547 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4548 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4549 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4550 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4551
4552 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:514553 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-14 06:25:224554 timestamp_ms += kFrameIntervalMs;
4555 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524556 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-14 06:25:224557 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 08:42:194558 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224559 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4560 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4561 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4562 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4563 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4564 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4565
4566 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:514567 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194568 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224569 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4570 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4571
mflodmancc3d4422017-08-03 15:27:514572 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224573}
4574
mflodmancc3d4422017-08-03 15:27:514575TEST_F(VideoStreamEncoderTest,
4576 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-14 06:25:224577 const int kWidth = 640;
4578 const int kHeight = 360;
4579 const int kFpsLimit = 15;
4580 const int64_t kFrameIntervalMs = 150;
4581 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 13:14:014582 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074583 DataRate::BitsPerSec(kTargetBitrateBps),
4584 DataRate::BitsPerSec(kTargetBitrateBps),
4585 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-14 06:25:224586
Taylor Brandstetter49fcc102018-05-16 21:20:414587 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:224588 AdaptingFrameForwarder source;
4589 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:414590 video_stream_encoder_->SetSource(&source,
4591 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:224592 timestamp_ms += kFrameIntervalMs;
4593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524594 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:194595 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4597 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4598 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4599 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4600 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4601 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4602
4603 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514604 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:224605 timestamp_ms += kFrameIntervalMs;
4606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524607 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224608 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4610 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4611 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4612 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4613 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4614 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4615
4616 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:514617 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224618 timestamp_ms += kFrameIntervalMs;
4619 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524620 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224621 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4622 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4623 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4624 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4625 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4626 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4627 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4628
4629 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514630 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-14 06:25:224631 timestamp_ms += kFrameIntervalMs;
4632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524633 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224634 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4635 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4637 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4639 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4640 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4641
4642 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514643 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224644 timestamp_ms += kFrameIntervalMs;
4645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524646 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 08:42:194647 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4649 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4650 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4651 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4652 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4653 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4654
4655 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:514656 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194657 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224658 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4659 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4660
mflodmancc3d4422017-08-03 15:27:514661 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224662}
4663
mflodmancc3d4422017-08-03 15:27:514664TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 13:53:484665 const int kFrameWidth = 1920;
4666 const int kFrameHeight = 1080;
4667 // 3/4 of 1920.
4668 const int kAdaptedFrameWidth = 1440;
4669 // 3/4 of 1080 rounded down to multiple of 4.
4670 const int kAdaptedFrameHeight = 808;
4671 const int kFramerate = 24;
4672
Erik Språng4c6ca302019-04-08 13:14:014673 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074674 DataRate::BitsPerSec(kTargetBitrateBps),
4675 DataRate::BitsPerSec(kTargetBitrateBps),
4676 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 13:53:484677 // Trigger reconfigure encoder (without resetting the entire instance).
4678 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:514679 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 13:53:484680 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4681 video_encoder_config.number_of_streams = 1;
4682 video_encoder_config.video_stream_factory =
4683 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 15:27:514684 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:474685 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:514686 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 13:53:484687
4688 video_source_.set_adaptation_enabled(true);
4689
4690 video_source_.IncomingCapturedFrame(
4691 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524692 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 13:53:484693
4694 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:514695 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 13:53:484696 video_source_.IncomingCapturedFrame(
4697 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524698 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 13:53:484699
mflodmancc3d4422017-08-03 15:27:514700 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 13:53:484701}
4702
mflodmancc3d4422017-08-03 15:27:514703TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 14:06:524704 const int kFrameWidth = 1280;
4705 const int kFrameHeight = 720;
4706 const int kLowFps = 2;
4707 const int kHighFps = 30;
4708
Erik Språng4c6ca302019-04-08 13:14:014709 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074710 DataRate::BitsPerSec(kTargetBitrateBps),
4711 DataRate::BitsPerSec(kTargetBitrateBps),
4712 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:524713
4714 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4715 max_framerate_ = kLowFps;
4716
4717 // Insert 2 seconds of 2fps video.
4718 for (int i = 0; i < kLowFps * 2; ++i) {
4719 video_source_.IncomingCapturedFrame(
4720 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4721 WaitForEncodedFrame(timestamp_ms);
4722 timestamp_ms += 1000 / kLowFps;
4723 }
4724
4725 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 13:14:014726 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074727 DataRate::BitsPerSec(kTargetBitrateBps),
4728 DataRate::BitsPerSec(kTargetBitrateBps),
4729 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 14:06:524730 video_source_.IncomingCapturedFrame(
4731 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4732 WaitForEncodedFrame(timestamp_ms);
4733 timestamp_ms += 1000 / kLowFps;
4734
4735 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4736
4737 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 08:48:484738 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 14:06:524739 const int kFrameIntervalMs = 1000 / kHighFps;
4740 max_framerate_ = kHighFps;
4741 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4742 video_source_.IncomingCapturedFrame(
4743 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4744 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4745 // be dropped if the encoder hans't been updated with the new higher target
4746 // framerate yet, causing it to overshoot the target bitrate and then
4747 // suffering the wrath of the media optimizer.
4748 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4749 timestamp_ms += kFrameIntervalMs;
4750 }
4751
4752 // Don expect correct measurement just yet, but it should be higher than
4753 // before.
4754 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4755
mflodmancc3d4422017-08-03 15:27:514756 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:524757}
4758
mflodmancc3d4422017-08-03 15:27:514759TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 14:06:524760 const int kFrameWidth = 1280;
4761 const int kFrameHeight = 720;
4762 const int kTargetBitrateBps = 1000000;
4763
4764 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 12:09:314765 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 13:14:014766 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074767 DataRate::BitsPerSec(kTargetBitrateBps),
4768 DataRate::BitsPerSec(kTargetBitrateBps),
4769 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 15:27:514770 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 14:06:524771
4772 // Insert a first video frame, causes another bitrate update.
4773 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4774 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4775 video_source_.IncomingCapturedFrame(
4776 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4777 WaitForEncodedFrame(timestamp_ms);
4778
4779 // Next, simulate video suspension due to pacer queue overrun.
Danil Chapovalovcad3e0e2020-02-17 17:46:074780 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
4781 DataRate::BitsPerSec(0),
4782 DataRate::BitsPerSec(0), 0, 1, 0);
sprang4847ae62017-06-27 14:06:524783
4784 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 08:48:484785 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 10:16:004786 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 14:06:524787
4788 // Bitrate observer should not be called.
4789 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4790 video_source_.IncomingCapturedFrame(
4791 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4792 ExpectDroppedFrame();
4793
mflodmancc3d4422017-08-03 15:27:514794 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:524795}
ilnik6b826ef2017-06-16 13:53:484796
Niels Möller4db138e2018-04-19 07:04:134797TEST_F(VideoStreamEncoderTest,
4798 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4799 const int kFrameWidth = 1280;
4800 const int kFrameHeight = 720;
4801 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 13:14:014802 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074803 DataRate::BitsPerSec(kTargetBitrateBps),
4804 DataRate::BitsPerSec(kTargetBitrateBps),
4805 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:134806 video_source_.IncomingCapturedFrame(
4807 CreateFrame(1, kFrameWidth, kFrameHeight));
4808 WaitForEncodedFrame(1);
4809 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4810 .low_encode_usage_threshold_percent,
4811 default_options.low_encode_usage_threshold_percent);
4812 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4813 .high_encode_usage_threshold_percent,
4814 default_options.high_encode_usage_threshold_percent);
4815 video_stream_encoder_->Stop();
4816}
4817
4818TEST_F(VideoStreamEncoderTest,
4819 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4820 const int kFrameWidth = 1280;
4821 const int kFrameHeight = 720;
4822 CpuOveruseOptions hardware_options;
4823 hardware_options.low_encode_usage_threshold_percent = 150;
4824 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 11:42:184825 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 07:04:134826
Erik Språng4c6ca302019-04-08 13:14:014827 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074828 DataRate::BitsPerSec(kTargetBitrateBps),
4829 DataRate::BitsPerSec(kTargetBitrateBps),
4830 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 07:04:134831 video_source_.IncomingCapturedFrame(
4832 CreateFrame(1, kFrameWidth, kFrameHeight));
4833 WaitForEncodedFrame(1);
4834 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4835 .low_encode_usage_threshold_percent,
4836 hardware_options.low_encode_usage_threshold_percent);
4837 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4838 .high_encode_usage_threshold_percent,
4839 hardware_options.high_encode_usage_threshold_percent);
4840 video_stream_encoder_->Stop();
4841}
4842
Niels Möller6bb5ab92019-01-11 10:11:104843TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4844 const int kFrameWidth = 320;
4845 const int kFrameHeight = 240;
4846 const int kFps = 30;
4847 const int kTargetBitrateBps = 120000;
4848 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4849
Erik Språng4c6ca302019-04-08 13:14:014850 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074851 DataRate::BitsPerSec(kTargetBitrateBps),
4852 DataRate::BitsPerSec(kTargetBitrateBps),
4853 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:104854
4855 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4856 max_framerate_ = kFps;
4857
4858 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4859 fake_encoder_.SimulateOvershoot(1.0);
4860 int num_dropped = 0;
4861 for (int i = 0; i < kNumFramesInRun; ++i) {
4862 video_source_.IncomingCapturedFrame(
4863 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4864 // Wait up to two frame durations for a frame to arrive.
4865 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4866 ++num_dropped;
4867 }
4868 timestamp_ms += 1000 / kFps;
4869 }
4870
Erik Språnga8d48ab2019-02-08 13:17:404871 // Framerate should be measured to be near the expected target rate.
4872 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4873
4874 // Frame drops should be within 5% of expected 0%.
4875 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:104876
4877 // Make encoder produce frames at double the expected bitrate during 3 seconds
4878 // of video, verify number of drops. Rate needs to be slightly changed in
4879 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 15:20:174880 double overshoot_factor = 2.0;
4881 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4882 // With bitrate adjuster, when need to overshoot even more to trigger
4883 // frame dropping.
4884 overshoot_factor *= 2;
4885 }
4886 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 14:37:334887 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074888 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4889 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4890 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:104891 num_dropped = 0;
4892 for (int i = 0; i < kNumFramesInRun; ++i) {
4893 video_source_.IncomingCapturedFrame(
4894 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4895 // Wait up to two frame durations for a frame to arrive.
4896 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4897 ++num_dropped;
4898 }
4899 timestamp_ms += 1000 / kFps;
4900 }
4901
Erik Språng4c6ca302019-04-08 13:14:014902 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074903 DataRate::BitsPerSec(kTargetBitrateBps),
4904 DataRate::BitsPerSec(kTargetBitrateBps),
4905 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 13:17:404906
4907 // Target framerate should be still be near the expected target, despite
4908 // the frame drops.
4909 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4910
4911 // Frame drops should be within 5% of expected 50%.
4912 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:104913
4914 video_stream_encoder_->Stop();
4915}
4916
4917TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4918 const int kFrameWidth = 320;
4919 const int kFrameHeight = 240;
4920 const int kActualInputFps = 24;
4921 const int kTargetBitrateBps = 120000;
4922
4923 ASSERT_GT(max_framerate_, kActualInputFps);
4924
4925 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4926 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 13:14:014927 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074928 DataRate::BitsPerSec(kTargetBitrateBps),
4929 DataRate::BitsPerSec(kTargetBitrateBps),
4930 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:104931
4932 // Insert 3 seconds of video, with an input fps lower than configured max.
4933 for (int i = 0; i < kActualInputFps * 3; ++i) {
4934 video_source_.IncomingCapturedFrame(
4935 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4936 // Wait up to two frame durations for a frame to arrive.
4937 WaitForEncodedFrame(timestamp_ms);
4938 timestamp_ms += 1000 / kActualInputFps;
4939 }
4940
4941 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4942
4943 video_stream_encoder_->Stop();
4944}
4945
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:264946TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4947 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 13:14:014948 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074949 DataRate::BitsPerSec(kTargetBitrateBps),
4950 DataRate::BitsPerSec(kTargetBitrateBps),
4951 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:264952
4953 fake_encoder_.BlockNextEncode();
4954 video_source_.IncomingCapturedFrame(
4955 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4956 WaitForEncodedFrame(1);
4957 // On the very first frame full update should be forced.
4958 rect = fake_encoder_.GetLastUpdateRect();
4959 EXPECT_EQ(rect.offset_x, 0);
4960 EXPECT_EQ(rect.offset_y, 0);
4961 EXPECT_EQ(rect.height, codec_height_);
4962 EXPECT_EQ(rect.width, codec_width_);
4963 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4964 // call to ContinueEncode.
4965 video_source_.IncomingCapturedFrame(
4966 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4967 ExpectDroppedFrame();
4968 video_source_.IncomingCapturedFrame(
4969 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4970 ExpectDroppedFrame();
4971 fake_encoder_.ContinueEncode();
4972 WaitForEncodedFrame(3);
4973 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4974 rect = fake_encoder_.GetLastUpdateRect();
4975 EXPECT_EQ(rect.offset_x, 1);
4976 EXPECT_EQ(rect.offset_y, 0);
4977 EXPECT_EQ(rect.width, 10);
4978 EXPECT_EQ(rect.height, 1);
4979
4980 video_source_.IncomingCapturedFrame(
4981 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4982 WaitForEncodedFrame(4);
4983 // Previous frame was encoded, so no accumulation should happen.
4984 rect = fake_encoder_.GetLastUpdateRect();
4985 EXPECT_EQ(rect.offset_x, 0);
4986 EXPECT_EQ(rect.offset_y, 0);
4987 EXPECT_EQ(rect.width, 1);
4988 EXPECT_EQ(rect.height, 1);
4989
4990 video_stream_encoder_->Stop();
4991}
4992
Erik Språngd7329ca2019-02-21 20:19:534993TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 13:14:014994 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:074995 DataRate::BitsPerSec(kTargetBitrateBps),
4996 DataRate::BitsPerSec(kTargetBitrateBps),
4997 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:534998
4999 // First frame is always keyframe.
5000 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5001 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 14:43:585002 EXPECT_THAT(
5003 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125004 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:535005
5006 // Insert delta frame.
5007 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5008 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 14:43:585009 EXPECT_THAT(
5010 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125011 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:535012
5013 // Request next frame be a key-frame.
5014 video_stream_encoder_->SendKeyFrame();
5015 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5016 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 14:43:585017 EXPECT_THAT(
5018 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125019 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:535020
5021 video_stream_encoder_->Stop();
5022}
5023
5024TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5025 // Setup simulcast with three streams.
5026 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 14:37:335027 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075028 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5029 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5030 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:535031 // Wait for all three layers before triggering event.
5032 sink_.SetNumExpectedLayers(3);
5033
5034 // First frame is always keyframe.
5035 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5036 WaitForEncodedFrame(1);
5037 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125038 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5039 VideoFrameType::kVideoFrameKey,
5040 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:535041
5042 // Insert delta frame.
5043 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5044 WaitForEncodedFrame(2);
5045 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125046 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5047 VideoFrameType::kVideoFrameDelta,
5048 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:535049
5050 // Request next frame be a key-frame.
5051 // Only first stream is configured to produce key-frame.
5052 video_stream_encoder_->SendKeyFrame();
5053 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5054 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 11:45:395055
5056 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5057 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 20:19:535058 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125059 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 11:45:395060 VideoFrameType::kVideoFrameKey,
5061 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:535062
5063 video_stream_encoder_->Stop();
5064}
5065
5066TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5067 // Configure internal source factory and setup test again.
5068 encoder_factory_.SetHasInternalSource(true);
5069 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 13:14:015070 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075071 DataRate::BitsPerSec(kTargetBitrateBps),
5072 DataRate::BitsPerSec(kTargetBitrateBps),
5073 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 20:19:535074
5075 // Call encoder directly, simulating internal source where encoded frame
5076 // callback in VideoStreamEncoder is called despite no OnFrame().
5077 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5078 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:585079 EXPECT_THAT(
5080 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125081 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:535082
Niels Möller8f7ce222019-03-21 14:43:585083 const std::vector<VideoFrameType> kDeltaFrame = {
5084 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:535085 // Need to set timestamp manually since manually for injected frame.
5086 VideoFrame frame = CreateFrame(101, nullptr);
5087 frame.set_timestamp(101);
5088 fake_encoder_.InjectFrame(frame, false);
5089 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:585090 EXPECT_THAT(
5091 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125092 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:535093
5094 // Request key-frame. The forces a dummy frame down into the encoder.
5095 fake_encoder_.ExpectNullFrame();
5096 video_stream_encoder_->SendKeyFrame();
5097 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:585098 EXPECT_THAT(
5099 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:125100 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:535101
5102 video_stream_encoder_->Stop();
5103}
Erik Språngb7cb7b52019-02-26 14:52:335104
5105TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5106 // Configure internal source factory and setup test again.
5107 encoder_factory_.SetHasInternalSource(true);
5108 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 13:14:015109 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075110 DataRate::BitsPerSec(kTargetBitrateBps),
5111 DataRate::BitsPerSec(kTargetBitrateBps),
5112 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 14:52:335113
5114 int64_t timestamp = 1;
5115 EncodedImage image;
Niels Möller4d504c72019-06-18 13:56:565116 image.SetEncodedData(
5117 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 14:52:335118 image.capture_time_ms_ = ++timestamp;
5119 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5120 const int64_t kEncodeFinishDelayMs = 10;
5121 image.timing_.encode_start_ms = timestamp;
5122 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5123 fake_encoder_.InjectEncodedImage(image);
5124 // Wait for frame without incrementing clock.
5125 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5126 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5127 // capture timestamp should be kEncodeFinishDelayMs in the past.
5128 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5129 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5130 kEncodeFinishDelayMs);
5131
5132 video_stream_encoder_->Stop();
5133}
Mirta Dvornicic28f0eb22019-05-28 14:30:165134
5135TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5136 // Configure internal source factory and setup test again.
5137 encoder_factory_.SetHasInternalSource(true);
5138 ResetEncoder("H264", 1, 1, 1, false);
5139
5140 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5141 image._frameType = VideoFrameType::kVideoFrameKey;
5142
5143 CodecSpecificInfo codec_specific_info;
5144 codec_specific_info.codecType = kVideoCodecH264;
5145
5146 RTPFragmentationHeader fragmentation;
5147 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5148 fragmentation.fragmentationOffset[0] = 4;
5149 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5150
5151 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5152 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5153
5154 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5155 testing::ElementsAreArray(optimal_sps));
5156 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5157 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5158 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5159 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5160
5161 video_stream_encoder_->Stop();
5162}
5163
5164TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5165 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5166 0x00, 0x00, 0x03, 0x03, 0xF4,
5167 0x05, 0x03, 0xC7, 0xC0};
5168
5169 // Configure internal source factory and setup test again.
5170 encoder_factory_.SetHasInternalSource(true);
5171 ResetEncoder("H264", 1, 1, 1, false);
5172
5173 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5174 image._frameType = VideoFrameType::kVideoFrameKey;
5175
5176 CodecSpecificInfo codec_specific_info;
5177 codec_specific_info.codecType = kVideoCodecH264;
5178
5179 RTPFragmentationHeader fragmentation;
5180 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5181 fragmentation.fragmentationOffset[0] = 4;
5182 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5183
5184 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5185 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5186
5187 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5188 testing::ElementsAreArray(optimal_sps));
5189 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5190 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5191 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5192 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5193
5194 video_stream_encoder_->Stop();
5195}
5196
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:145197TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5198 const int kFrameWidth = 1280;
5199 const int kFrameHeight = 720;
5200 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5201
5202 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075203 DataRate::BitsPerSec(kTargetBitrateBps),
5204 DataRate::BitsPerSec(kTargetBitrateBps),
5205 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:145206 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5207
5208 // Insert a first video frame. It should be dropped because of downscale in
5209 // resolution.
5210 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5211 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5212 frame.set_rotation(kVideoRotation_270);
5213 video_source_.IncomingCapturedFrame(frame);
5214
5215 ExpectDroppedFrame();
5216
5217 // Second frame is downscaled.
5218 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5219 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5220 frame.set_rotation(kVideoRotation_90);
5221 video_source_.IncomingCapturedFrame(frame);
5222
5223 WaitForEncodedFrame(timestamp_ms);
5224 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5225
5226 // Insert another frame, also downscaled.
5227 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5228 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5229 frame.set_rotation(kVideoRotation_180);
5230 video_source_.IncomingCapturedFrame(frame);
5231
5232 WaitForEncodedFrame(timestamp_ms);
5233 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5234
5235 video_stream_encoder_->Stop();
5236}
5237
Erik Språng5056af02019-09-02 13:53:115238TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5239 const int kFrameWidth = 320;
5240 const int kFrameHeight = 180;
5241
5242 // Initial rate.
5243 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075244 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5245 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5246 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 13:53:115247 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:325248 /*rtt_ms=*/0,
5249 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:115250
5251 // Insert a first video frame so that encoder gets configured.
5252 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5253 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5254 frame.set_rotation(kVideoRotation_270);
5255 video_source_.IncomingCapturedFrame(frame);
5256 WaitForEncodedFrame(timestamp_ms);
5257
5258 // Set a target rate below the minimum allowed by the codec settings.
5259 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 17:46:075260 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5261 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Erik Språng5056af02019-09-02 13:53:115262 video_stream_encoder_->OnBitrateUpdated(
5263 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 11:36:555264 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 13:53:115265 /*link_allocation=*/target_rate,
5266 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:325267 /*rtt_ms=*/0,
5268 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 13:53:115269 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5270
5271 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5272 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5273 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 17:46:075274 DataRate allocation_sum =
5275 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 13:53:115276 EXPECT_EQ(min_rate, allocation_sum);
5277 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5278
5279 video_stream_encoder_->Stop();
5280}
5281
Rasmus Brandt5cad55b2019-12-19 08:47:115282TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Evan Shrubsolee32ae4f2019-09-25 10:50:235283 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075284 DataRate::BitsPerSec(kTargetBitrateBps),
5285 DataRate::BitsPerSec(kTargetBitrateBps),
5286 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 10:50:235287 // Capture a frame and wait for it to synchronize with the encoder thread.
5288 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5289 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5290 WaitForEncodedFrame(1);
5291
5292 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5293 ASSERT_TRUE(prev_rate_settings.has_value());
5294 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5295 kDefaultFramerate);
5296
5297 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5298 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5299 timestamp_ms += 1000 / kDefaultFramerate;
5300 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5301 WaitForEncodedFrame(timestamp_ms);
5302 }
5303 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5304 kDefaultFramerate);
5305 // Capture larger frame to trigger a reconfigure.
5306 codec_height_ *= 2;
5307 codec_width_ *= 2;
5308 timestamp_ms += 1000 / kDefaultFramerate;
5309 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5310 WaitForEncodedFrame(timestamp_ms);
5311
5312 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5313 auto current_rate_settings =
5314 fake_encoder_.GetAndResetLastRateControlSettings();
5315 // Ensure we have actually reconfigured twice
5316 // The rate settings should have been set again even though
5317 // they haven't changed.
5318 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 07:55:035319 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 10:50:235320
5321 video_stream_encoder_->Stop();
5322}
5323
philipeld9cc8c02019-09-16 12:53:405324struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
5325 MOCK_METHOD0(RequestEncoderFallback, void());
5326 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
philipel9b058032020-02-10 10:30:005327 MOCK_METHOD1(RequestEncoderSwitch,
5328 void(const webrtc::SdpVideoFormat& format));
philipeld9cc8c02019-09-16 12:53:405329};
5330
5331TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5332 constexpr int kDontCare = 100;
5333
5334 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5335 video_send_config_.encoder_settings.encoder_switch_request_callback =
5336 &switch_callback;
5337 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5338 encoder_config.codec_type = kVideoCodecVP8;
5339 webrtc::test::ScopedFieldTrials field_trial(
5340 "WebRTC-NetworkCondition-EncoderSwitch/"
5341 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5342 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5343
5344 // Reset encoder for new configuration to take effect.
5345 ConfigureEncoder(std::move(encoder_config));
5346
5347 // Send one frame to trigger ReconfigureEncoder.
5348 video_source_.IncomingCapturedFrame(
5349 CreateFrame(kDontCare, kDontCare, kDontCare));
5350
5351 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 10:30:005352 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5353 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 12:53:405354 Field(&Config::param, "ping"),
philipel9b058032020-02-10 10:30:005355 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 12:53:405356
5357 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075358 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5359 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5360 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 12:53:405361 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:325362 /*rtt_ms=*/0,
5363 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 12:53:405364
5365 video_stream_encoder_->Stop();
5366}
5367
Mirta Dvornicic5ed40cf2020-02-21 15:35:515368TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5369 constexpr int kDontCare = 100;
5370
5371 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5372 video_send_config_.encoder_settings.encoder_switch_request_callback =
5373 &switch_callback;
5374 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5375 encoder_config.codec_type = kVideoCodecVP8;
5376 webrtc::test::ScopedFieldTrials field_trial(
5377 "WebRTC-NetworkCondition-EncoderSwitch/"
5378 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5379 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5380
5381 // Reset encoder for new configuration to take effect.
5382 ConfigureEncoder(std::move(encoder_config));
5383
5384 // Send one frame to trigger ReconfigureEncoder.
5385 video_source_.IncomingCapturedFrame(
5386 CreateFrame(kDontCare, kDontCare, kDontCare));
5387
5388 using Config = EncoderSwitchRequestCallback::Config;
5389 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5390 .Times(0);
5391
5392 video_stream_encoder_->OnBitrateUpdated(
5393 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5394 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5395 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5396 /*fraction_lost=*/0,
5397 /*rtt_ms=*/0,
5398 /*cwnd_reduce_ratio=*/0);
5399
5400 video_stream_encoder_->Stop();
5401}
5402
philipeld9cc8c02019-09-16 12:53:405403TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5404 constexpr int kSufficientBitrateToNotDrop = 1000;
5405 constexpr int kHighRes = 500;
5406 constexpr int kLowRes = 100;
5407
5408 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5409 video_send_config_.encoder_settings.encoder_switch_request_callback =
5410 &switch_callback;
5411 webrtc::test::ScopedFieldTrials field_trial(
5412 "WebRTC-NetworkCondition-EncoderSwitch/"
5413 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5414 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5415 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5416 encoder_config.codec_type = kVideoCodecH264;
5417
5418 // Reset encoder for new configuration to take effect.
5419 ConfigureEncoder(std::move(encoder_config));
5420
5421 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5422 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5423 // not fail.
5424 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075425 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5426 /*stable_target_bitrate=*/
5427 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5428 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 12:53:405429 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:325430 /*rtt_ms=*/0,
5431 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 12:53:405432
5433 // Send one frame to trigger ReconfigureEncoder.
5434 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5435 WaitForEncodedFrame(1);
5436
5437 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 10:30:005438 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5439 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 12:53:405440 Field(&Config::param, "ping"),
philipel9b058032020-02-10 10:30:005441 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 12:53:405442
5443 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5444 WaitForEncodedFrame(2);
5445
5446 video_stream_encoder_->Stop();
5447}
5448
philipel9b058032020-02-10 10:30:005449TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5450 constexpr int kDontCare = 100;
5451 StrictMock<MockEncoderSelector> encoder_selector;
5452 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5453 &fake_encoder_, &encoder_selector);
5454 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5455
5456 // Reset encoder for new configuration to take effect.
5457 ConfigureEncoder(video_encoder_config_.Copy());
5458
5459 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5460
5461 video_source_.IncomingCapturedFrame(
5462 CreateFrame(kDontCare, kDontCare, kDontCare));
5463 video_stream_encoder_->Stop();
5464
5465 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5466 // to it's factory, so in order for the encoder instance in the
5467 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5468 // reset the |video_stream_encoder_| here.
5469 video_stream_encoder_.reset();
5470}
5471
5472TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5473 constexpr int kDontCare = 100;
5474
5475 NiceMock<MockEncoderSelector> encoder_selector;
5476 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5477 video_send_config_.encoder_settings.encoder_switch_request_callback =
5478 &switch_callback;
5479 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5480 &fake_encoder_, &encoder_selector);
5481 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5482
5483 // Reset encoder for new configuration to take effect.
5484 ConfigureEncoder(video_encoder_config_.Copy());
5485
Mirta Dvornicic4f34d782020-02-26 12:01:195486 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 10:30:005487 .WillByDefault(Return(SdpVideoFormat("AV1")));
5488 EXPECT_CALL(switch_callback,
5489 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5490 Field(&SdpVideoFormat::name, "AV1"))));
5491
5492 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075493 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5494 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5495 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 10:30:005496 /*fraction_lost=*/0,
5497 /*rtt_ms=*/0,
5498 /*cwnd_reduce_ratio=*/0);
5499
5500 video_stream_encoder_->Stop();
5501}
5502
5503TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5504 constexpr int kSufficientBitrateToNotDrop = 1000;
5505 constexpr int kDontCare = 100;
5506
5507 NiceMock<MockVideoEncoder> video_encoder;
5508 NiceMock<MockEncoderSelector> encoder_selector;
5509 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5510 video_send_config_.encoder_settings.encoder_switch_request_callback =
5511 &switch_callback;
5512 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5513 &video_encoder, &encoder_selector);
5514 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5515
5516 // Reset encoder for new configuration to take effect.
5517 ConfigureEncoder(video_encoder_config_.Copy());
5518
5519 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5520 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5521 // not fail.
5522 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075523 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5524 /*stable_target_bitrate=*/
5525 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5526 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 10:30:005527 /*fraction_lost=*/0,
5528 /*rtt_ms=*/0,
5529 /*cwnd_reduce_ratio=*/0);
5530
5531 ON_CALL(video_encoder, Encode(_, _))
5532 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5533 ON_CALL(encoder_selector, OnEncoderBroken())
5534 .WillByDefault(Return(SdpVideoFormat("AV2")));
5535
5536 rtc::Event encode_attempted;
5537 EXPECT_CALL(switch_callback,
5538 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5539 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5540 EXPECT_EQ(format.name, "AV2");
5541 encode_attempted.Set();
5542 });
5543
5544 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5545 encode_attempted.Wait(3000);
5546
5547 video_stream_encoder_->Stop();
5548
5549 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5550 // to it's factory, so in order for the encoder instance in the
5551 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5552 // reset the |video_stream_encoder_| here.
5553 video_stream_encoder_.reset();
5554}
5555
Evan Shrubsole7c079f62019-09-26 07:55:035556TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:115557 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 07:55:035558 const int kFrameWidth = 320;
5559 const int kFrameHeight = 180;
5560
5561 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:075562 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 07:55:035563 video_stream_encoder_->OnBitrateUpdated(
5564 /*target_bitrate=*/rate,
5565 /*stable_target_bitrate=*/rate,
5566 /*link_allocation=*/rate,
5567 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:325568 /*rtt_ms=*/0,
5569 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:035570
5571 // Insert a first video frame so that encoder gets configured.
5572 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5573 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5574 frame.set_rotation(kVideoRotation_270);
5575 video_source_.IncomingCapturedFrame(frame);
5576 WaitForEncodedFrame(timestamp_ms);
5577 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5578
5579 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 17:46:075580 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 07:55:035581 video_stream_encoder_->OnBitrateUpdated(
5582 /*target_bitrate=*/new_stable_rate,
5583 /*stable_target_bitrate=*/new_stable_rate,
5584 /*link_allocation=*/rate,
5585 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:325586 /*rtt_ms=*/0,
5587 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:035588 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5589 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5590 video_stream_encoder_->Stop();
5591}
5592
5593TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 08:47:115594 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 07:55:035595 const int kFrameWidth = 320;
5596 const int kFrameHeight = 180;
5597
5598 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:075599 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 07:55:035600 video_stream_encoder_->OnBitrateUpdated(
5601 /*target_bitrate=*/rate,
5602 /*stable_target_bitrate=*/rate,
5603 /*link_allocation=*/rate,
5604 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:325605 /*rtt_ms=*/0,
5606 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:035607
5608 // Insert a first video frame so that encoder gets configured.
5609 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5610 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5611 frame.set_rotation(kVideoRotation_270);
5612 video_source_.IncomingCapturedFrame(frame);
5613 WaitForEncodedFrame(timestamp_ms);
5614 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5615
5616 // Set a higher target rate without changing the link_allocation. Should not
5617 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 17:46:075618 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 07:55:035619 video_stream_encoder_->OnBitrateUpdated(
5620 /*target_bitrate=*/rate,
5621 /*stable_target_bitrate=*/new_stable_rate,
5622 /*link_allocation=*/rate,
5623 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 13:29:325624 /*rtt_ms=*/0,
5625 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 07:55:035626 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5627 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5628 video_stream_encoder_->Stop();
5629}
5630
Ilya Nikolaevskiy648b9d72019-12-03 15:54:175631TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5632 test::ScopedFieldTrials field_trials(
5633 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5634 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5635 const int kFramerateFps = 30;
5636 const int kWidth = 1920;
5637 const int kHeight = 1080;
5638 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5639 // Works on screenshare mode.
5640 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5641 // We rely on the automatic resolution adaptation, but we handle framerate
5642 // adaptation manually by mocking the stats proxy.
5643 video_source_.set_adaptation_enabled(true);
5644
5645 // BALANCED degradation preference is required for this feature.
5646 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 17:46:075647 DataRate::BitsPerSec(kTargetBitrateBps),
5648 DataRate::BitsPerSec(kTargetBitrateBps),
5649 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 15:54:175650 video_stream_encoder_->SetSource(&video_source_,
5651 webrtc::DegradationPreference::BALANCED);
5652 VerifyNoLimitation(video_source_.sink_wants());
5653
5654 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5655 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5656
5657 // Pass enough frames with the full update to trigger animation detection.
5658 for (int i = 0; i < kNumFrames; ++i) {
5659 int64_t timestamp_ms =
5660 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5661 frame.set_ntp_time_ms(timestamp_ms);
5662 frame.set_timestamp_us(timestamp_ms * 1000);
5663 video_source_.IncomingCapturedFrame(frame);
5664 WaitForEncodedFrame(timestamp_ms);
5665 }
5666
5667 // Resolution should be limited.
5668 rtc::VideoSinkWants expected;
5669 expected.max_framerate_fps = kFramerateFps;
5670 expected.max_pixel_count = 1280 * 720 + 1;
5671 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5672
5673 // Pass one frame with no known update.
5674 // Resolution cap should be removed immediately.
5675 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5676 frame.set_ntp_time_ms(timestamp_ms);
5677 frame.set_timestamp_us(timestamp_ms * 1000);
5678 frame.clear_update_rect();
5679
5680 video_source_.IncomingCapturedFrame(frame);
5681 WaitForEncodedFrame(timestamp_ms);
5682
5683 // Resolution should be unlimited now.
5684 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5685
5686 video_stream_encoder_->Stop();
5687}
5688
perkj26091b12016-09-01 08:17:405689} // namespace webrtc