blob: 73cd326c8223a00324e5d8464ef205b785b01efa [file] [log] [blame]
perkj26091b12016-09-01 08:17:401/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 08:30:3111#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 15:24:5913#include <algorithm>
perkj803d97f2016-11-01 18:45:4614#include <limits>
Danil Chapovalovd3ba2362019-04-10 15:01:2315#include <memory>
Per512ecb32016-09-23 13:52:0616#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 14:30:1618#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 15:01:2319#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 09:20:0920#include "api/test/mock_fec_controller_override.h"
Jiawei Ouc2ebe212018-11-08 18:02:5621#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3122#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 16:10:5723#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 12:57:5724#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 08:30:3125#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 10:56:2026#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 14:30:1627#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 13:59:1228#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 17:11:0029#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 17:32:3730#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3131#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Åsa Perssonc29cb2c2019-03-25 11:06:5932#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 17:11:0033#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3134#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 17:11:0035#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 15:20:1736#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 06:51:1037#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3138#include "system_wrappers/include/sleep.h"
39#include "test/encoder_settings.h"
40#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 08:51:4041#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 14:59:0042#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3143#include "test/gmock.h"
44#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 07:07:2445#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3146#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 08:17:4047
48namespace webrtc {
49
sprangb1ca0732017-02-01 16:38:1250using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 14:24:0251using ::testing::_;
philipeld9cc8c02019-09-16 12:53:4052using ::testing::AllOf;
53using ::testing::Field;
54using ::testing::StrictMock;
kthelgason876222f2016-11-29 09:44:1155
perkj803d97f2016-11-01 18:45:4656namespace {
Åsa Persson8c1bf952018-09-13 08:42:1957const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 14:56:0058const int kQpLow = 1;
59const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 08:42:1960const int kMinFramerateFps = 2;
61const int kMinBalancedFramerateFps = 7;
62const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 12:37:0063const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 20:19:5364const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 12:18:3465const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 20:19:5366const uint32_t kSimulcastTargetBitrateBps = 3150000;
67const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 15:02:2268const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 11:21:0769const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 08:42:1970const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 08:48:4871const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 11:12:1972const VideoEncoder::ResolutionBitrateLimits
73 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
74const VideoEncoder::ResolutionBitrateLimits
75 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 12:37:0076
Mirta Dvornicic28f0eb22019-05-28 14:30:1677uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
78 0x00, 0x00, 0x03, 0x03, 0xF4,
79 0x05, 0x03, 0xC7, 0xE0, 0x1B,
80 0x41, 0x10, 0x8D, 0x00};
81
perkj803d97f2016-11-01 18:45:4682class TestBuffer : public webrtc::I420Buffer {
83 public:
84 TestBuffer(rtc::Event* event, int width, int height)
85 : I420Buffer(width, height), event_(event) {}
86
87 private:
88 friend class rtc::RefCountedObject<TestBuffer>;
89 ~TestBuffer() override {
90 if (event_)
91 event_->Set();
92 }
93 rtc::Event* const event_;
94};
95
Noah Richards51db4212019-06-12 13:59:1296// A fake native buffer that can't be converted to I420.
97class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
98 public:
99 FakeNativeBuffer(rtc::Event* event, int width, int height)
100 : event_(event), width_(width), height_(height) {}
101 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
102 int width() const override { return width_; }
103 int height() const override { return height_; }
104 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
105 return nullptr;
106 }
107
108 private:
109 friend class rtc::RefCountedObject<FakeNativeBuffer>;
110 ~FakeNativeBuffer() override {
111 if (event_)
112 event_->Set();
113 }
114 rtc::Event* const event_;
115 const int width_;
116 const int height_;
117};
118
Niels Möller7dc26b72017-12-06 09:27:48119class CpuOveruseDetectorProxy : public OveruseFrameDetector {
120 public:
Niels Möllerd1f7eb62018-03-28 14:40:58121 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
122 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 09:27:48123 last_target_framerate_fps_(-1) {}
124 virtual ~CpuOveruseDetectorProxy() {}
125
126 void OnTargetFramerateUpdated(int framerate_fps) override {
127 rtc::CritScope cs(&lock_);
128 last_target_framerate_fps_ = framerate_fps;
129 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
130 }
131
132 int GetLastTargetFramerate() {
133 rtc::CritScope cs(&lock_);
134 return last_target_framerate_fps_;
135 }
136
Niels Möller4db138e2018-04-19 07:04:13137 CpuOveruseOptions GetOptions() { return options_; }
138
Niels Möller7dc26b72017-12-06 09:27:48139 private:
140 rtc::CriticalSection lock_;
141 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
142};
143
mflodmancc3d4422017-08-03 15:27:51144class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 18:45:46145 public:
Niels Möller213618e2018-07-24 07:29:58146 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 15:01:23147 const VideoStreamEncoderSettings& settings,
148 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 17:30:41149 : VideoStreamEncoder(Clock::GetRealTimeClock(),
150 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 13:03:05151 stats_proxy,
152 settings,
Yves Gerey665174f2018-06-19 13:03:05153 std::unique_ptr<OveruseFrameDetector>(
154 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 10:50:20155 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 15:01:23156 task_queue_factory) {}
perkj803d97f2016-11-01 18:45:46157
sprangb1ca0732017-02-01 16:38:12158 void PostTaskAndWait(bool down, AdaptReason reason) {
Åsa Perssonb67c44c2019-09-24 13:25:32159 PostTaskAndWait(down, reason, /*expected_results=*/true);
160 }
161
162 void PostTaskAndWait(bool down, AdaptReason reason, bool expected_results) {
Niels Möllerc572ff32018-11-07 07:43:50163 rtc::Event event;
Åsa Perssonb67c44c2019-09-24 13:25:32164 encoder_queue()->PostTask([this, &event, reason, down, expected_results] {
Åsa Perssonf5e5d252019-08-16 15:24:59165 if (down)
Åsa Perssonb67c44c2019-09-24 13:25:32166 EXPECT_EQ(expected_results, AdaptDown(reason));
Åsa Perssonf5e5d252019-08-16 15:24:59167 else
168 AdaptUp(reason);
perkj803d97f2016-11-01 18:45:46169 event.Set();
170 });
perkj070ba852017-02-16 23:46:27171 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 18:45:46172 }
173
kthelgason2fc52542017-03-03 08:24:41174 // This is used as a synchronisation mechanism, to make sure that the
175 // encoder queue is not blocked before we start sending it frames.
176 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 07:43:50177 rtc::Event event;
Yves Gerey665174f2018-06-19 13:03:05178 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 08:24:41179 ASSERT_TRUE(event.Wait(5000));
180 }
181
Åsa Perssonb67c44c2019-09-24 13:25:32182 void TriggerCpuOveruse() {
183 PostTaskAndWait(/*down=*/true, AdaptReason::kCpu);
184 }
kthelgason876222f2016-11-29 09:44:11185
Åsa Perssonb67c44c2019-09-24 13:25:32186 void TriggerCpuNormalUsage() {
187 PostTaskAndWait(/*down=*/false, AdaptReason::kCpu);
188 }
kthelgason876222f2016-11-29 09:44:11189
Åsa Perssonb67c44c2019-09-24 13:25:32190 void TriggerQualityLow() {
191 PostTaskAndWait(/*down=*/true, AdaptReason::kQuality);
192 }
kthelgason876222f2016-11-29 09:44:11193
Åsa Perssonb67c44c2019-09-24 13:25:32194 void TriggerQualityLowExpectFalse() {
195 PostTaskAndWait(/*down=*/true, AdaptReason::kQuality,
196 /*expected_results=*/false);
197 }
198
199 void TriggerQualityHigh() {
200 PostTaskAndWait(/*down=*/false, AdaptReason::kQuality);
201 }
sprangfda496a2017-06-15 11:21:07202
Niels Möller7dc26b72017-12-06 09:27:48203 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 18:45:46204};
205
asapersson5f7226f2016-11-25 12:37:00206class VideoStreamFactory
207 : public VideoEncoderConfig::VideoStreamFactoryInterface {
208 public:
sprangfda496a2017-06-15 11:21:07209 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
210 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 12:37:00211 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 11:21:07212 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 12:37:00213 }
214
215 private:
216 std::vector<VideoStream> CreateEncoderStreams(
217 int width,
218 int height,
219 const VideoEncoderConfig& encoder_config) override {
220 std::vector<VideoStream> streams =
221 test::CreateVideoStreams(width, height, encoder_config);
222 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 14:11:29223 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 11:21:07224 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 12:37:00225 }
226 return streams;
227 }
sprangfda496a2017-06-15 11:21:07228
asapersson5f7226f2016-11-25 12:37:00229 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 11:21:07230 const int framerate_;
asapersson5f7226f2016-11-25 12:37:00231};
232
Noah Richards51db4212019-06-12 13:59:12233// Simulates simulcast behavior and makes highest stream resolutions divisible
234// by 4.
235class CroppingVideoStreamFactory
236 : public VideoEncoderConfig::VideoStreamFactoryInterface {
237 public:
238 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
239 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
240 EXPECT_GT(num_temporal_layers, 0u);
241 EXPECT_GT(framerate, 0);
242 }
243
244 private:
245 std::vector<VideoStream> CreateEncoderStreams(
246 int width,
247 int height,
248 const VideoEncoderConfig& encoder_config) override {
249 std::vector<VideoStream> streams = test::CreateVideoStreams(
250 width - width % 4, height - height % 4, encoder_config);
251 for (VideoStream& stream : streams) {
252 stream.num_temporal_layers = num_temporal_layers_;
253 stream.max_framerate = framerate_;
254 }
255 return streams;
256 }
257
258 const size_t num_temporal_layers_;
259 const int framerate_;
260};
261
sprangb1ca0732017-02-01 16:38:12262class AdaptingFrameForwarder : public test::FrameForwarder {
263 public:
264 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 12:51:49265 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 16:38:12266
267 void set_adaptation_enabled(bool enabled) {
268 rtc::CritScope cs(&crit_);
269 adaptation_enabled_ = enabled;
270 }
271
asaperssonfab67072017-04-04 12:51:49272 bool adaption_enabled() const {
sprangb1ca0732017-02-01 16:38:12273 rtc::CritScope cs(&crit_);
274 return adaptation_enabled_;
275 }
276
asapersson09f05612017-05-16 06:40:18277 rtc::VideoSinkWants last_wants() const {
278 rtc::CritScope cs(&crit_);
279 return last_wants_;
280 }
281
Danil Chapovalovb9b146c2018-06-15 10:28:07282 absl::optional<int> last_sent_width() const { return last_width_; }
283 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-09 01:04:29284
sprangb1ca0732017-02-01 16:38:12285 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
286 int cropped_width = 0;
287 int cropped_height = 0;
288 int out_width = 0;
289 int out_height = 0;
sprangc5d62e22017-04-03 06:53:04290 if (adaption_enabled()) {
291 if (adapter_.AdaptFrameResolution(
292 video_frame.width(), video_frame.height(),
293 video_frame.timestamp_us() * 1000, &cropped_width,
294 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 22:49:37295 VideoFrame adapted_frame =
296 VideoFrame::Builder()
297 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
298 nullptr, out_width, out_height))
299 .set_timestamp_rtp(99)
300 .set_timestamp_ms(99)
301 .set_rotation(kVideoRotation_0)
302 .build();
sprangc5d62e22017-04-03 06:53:04303 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 15:54:17304 if (video_frame.has_update_rect()) {
305 adapted_frame.set_update_rect(
306 video_frame.update_rect().ScaleWithFrame(
307 video_frame.width(), video_frame.height(), 0, 0,
308 video_frame.width(), video_frame.height(), out_width,
309 out_height));
310 }
sprangc5d62e22017-04-03 06:53:04311 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-09 01:04:29312 last_width_.emplace(adapted_frame.width());
313 last_height_.emplace(adapted_frame.height());
314 } else {
Danil Chapovalovb9b146c2018-06-15 10:28:07315 last_width_ = absl::nullopt;
316 last_height_ = absl::nullopt;
sprangc5d62e22017-04-03 06:53:04317 }
sprangb1ca0732017-02-01 16:38:12318 } else {
319 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-09 01:04:29320 last_width_.emplace(video_frame.width());
321 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 16:38:12322 }
323 }
324
325 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
326 const rtc::VideoSinkWants& wants) override {
327 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-16 06:40:18328 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 15:56:01329 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 16:38:12330 test::FrameForwarder::AddOrUpdateSink(sink, wants);
331 }
sprangb1ca0732017-02-01 16:38:12332 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 11:17:22333 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
334 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 10:28:07335 absl::optional<int> last_width_;
336 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 16:38:12337};
sprangc5d62e22017-04-03 06:53:04338
Niels Möller213618e2018-07-24 07:29:58339// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-03 06:53:04340class MockableSendStatisticsProxy : public SendStatisticsProxy {
341 public:
342 MockableSendStatisticsProxy(Clock* clock,
343 const VideoSendStream::Config& config,
344 VideoEncoderConfig::ContentType content_type)
345 : SendStatisticsProxy(clock, config, content_type) {}
346
347 VideoSendStream::Stats GetStats() override {
348 rtc::CritScope cs(&lock_);
349 if (mock_stats_)
350 return *mock_stats_;
351 return SendStatisticsProxy::GetStats();
352 }
353
Niels Möller213618e2018-07-24 07:29:58354 int GetInputFrameRate() const override {
355 rtc::CritScope cs(&lock_);
356 if (mock_stats_)
357 return mock_stats_->input_frame_rate;
358 return SendStatisticsProxy::GetInputFrameRate();
359 }
sprangc5d62e22017-04-03 06:53:04360 void SetMockStats(const VideoSendStream::Stats& stats) {
361 rtc::CritScope cs(&lock_);
362 mock_stats_.emplace(stats);
363 }
364
365 void ResetMockStats() {
366 rtc::CritScope cs(&lock_);
367 mock_stats_.reset();
368 }
369
370 private:
371 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 10:28:07372 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-03 06:53:04373};
374
sprang4847ae62017-06-27 14:06:52375class MockBitrateObserver : public VideoBitrateAllocationObserver {
376 public:
Erik Språng566124a2018-04-23 10:32:22377 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 14:06:52378};
379
perkj803d97f2016-11-01 18:45:46380} // namespace
381
mflodmancc3d4422017-08-03 15:27:51382class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 08:17:40383 public:
384 static const int kDefaultTimeoutMs = 30 * 1000;
385
mflodmancc3d4422017-08-03 15:27:51386 VideoStreamEncoderTest()
perkj26091b12016-09-01 08:17:40387 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-03 06:45:26388 codec_width_(320),
389 codec_height_(240),
Åsa Persson8c1bf952018-09-13 08:42:19390 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 15:01:23391 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 08:17:40392 fake_encoder_(),
Niels Möller4db138e2018-04-19 07:04:13393 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-03 06:53:04394 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 18:45:46395 Clock::GetRealTimeClock(),
396 video_send_config_,
397 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 08:17:40398 sink_(&fake_encoder_) {}
399
400 void SetUp() override {
perkj803d97f2016-11-01 18:45:46401 metrics::Reset();
perkj26091b12016-09-01 08:17:40402 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 07:04:13403 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 18:02:56404 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 12:18:34405 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 13:36:51406 video_send_config_.rtp.payload_name = "FAKE";
407 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 08:17:40408
Per512ecb32016-09-23 13:52:06409 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51410 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 14:06:52411 video_encoder_config.video_stream_factory =
412 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 15:41:30413 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 14:06:52414
415 // Framerate limit is specified by the VideoStreamFactory.
416 std::vector<VideoStream> streams =
417 video_encoder_config.video_stream_factory->CreateEncoderStreams(
418 codec_width_, codec_height_, video_encoder_config);
419 max_framerate_ = streams[0].max_framerate;
Sebastian Jansson40889f32019-04-17 10:11:20420 fake_clock_.SetTime(Timestamp::us(1234));
sprang4847ae62017-06-27 14:06:52421
Niels Möllerf1338562018-04-26 07:51:47422 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 12:37:00423 }
424
Niels Möllerf1338562018-04-26 07:51:47425 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 15:27:51426 if (video_stream_encoder_)
427 video_stream_encoder_->Stop();
428 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 15:01:23429 stats_proxy_.get(), video_send_config_.encoder_settings,
430 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 15:27:51431 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
432 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:41433 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 15:27:51434 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
435 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:47436 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:51437 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 12:37:00438 }
439
440 void ResetEncoder(const std::string& payload_name,
441 size_t num_streams,
442 size_t num_temporal_layers,
emircanbbcc3562017-08-18 07:28:40443 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 14:06:52444 bool screenshare) {
Niels Möller259a4972018-04-05 13:36:51445 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 12:37:00446
447 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:51448 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 12:37:00449 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 20:19:53450 video_encoder_config.max_bitrate_bps =
451 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 12:37:00452 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 11:21:07453 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
454 kDefaultFramerate);
sprang4847ae62017-06-27 14:06:52455 video_encoder_config.content_type =
456 screenshare ? VideoEncoderConfig::ContentType::kScreen
457 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 07:28:40458 if (payload_name == "VP9") {
459 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
460 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
461 video_encoder_config.encoder_specific_settings =
462 new rtc::RefCountedObject<
463 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
464 }
Niels Möllerf1338562018-04-26 07:51:47465 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 08:17:40466 }
467
sprang57c2fff2017-01-16 14:24:02468 VideoFrame CreateFrame(int64_t ntp_time_ms,
469 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 22:49:37470 VideoFrame frame =
471 VideoFrame::Builder()
472 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
473 destruction_event, codec_width_, codec_height_))
474 .set_timestamp_rtp(99)
475 .set_timestamp_ms(99)
476 .set_rotation(kVideoRotation_0)
477 .build();
sprang57c2fff2017-01-16 14:24:02478 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 08:17:40479 return frame;
480 }
481
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26482 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
483 rtc::Event* destruction_event,
484 int offset_x) const {
485 VideoFrame frame =
486 VideoFrame::Builder()
487 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
488 destruction_event, codec_width_, codec_height_))
489 .set_timestamp_rtp(99)
490 .set_timestamp_ms(99)
491 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 09:34:12492 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26493 .build();
494 frame.set_ntp_time_ms(ntp_time_ms);
495 return frame;
496 }
497
sprang57c2fff2017-01-16 14:24:02498 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 22:49:37499 VideoFrame frame =
500 VideoFrame::Builder()
501 .set_video_frame_buffer(
502 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
503 .set_timestamp_rtp(99)
504 .set_timestamp_ms(99)
505 .set_rotation(kVideoRotation_0)
506 .build();
sprang57c2fff2017-01-16 14:24:02507 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-03 06:53:04508 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 18:45:46509 return frame;
510 }
511
Noah Richards51db4212019-06-12 13:59:12512 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
513 rtc::Event* destruction_event,
514 int width,
515 int height) const {
516 VideoFrame frame =
517 VideoFrame::Builder()
518 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
519 destruction_event, width, height))
520 .set_timestamp_rtp(99)
521 .set_timestamp_ms(99)
522 .set_rotation(kVideoRotation_0)
523 .build();
524 frame.set_ntp_time_ms(ntp_time_ms);
525 return frame;
526 }
527
528 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
529 rtc::Event* destruction_event) const {
530 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
531 codec_height_);
532 }
533
Åsa Perssonc29cb2c2019-03-25 11:06:59534 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
535 MockBitrateObserver bitrate_observer;
536 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
537
538 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
539 .Times(1);
Florent Castellia8336d32019-09-09 11:36:55540 video_stream_encoder_->OnBitrateUpdated(
541 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
542 DataRate::bps(kTargetBitrateBps), 0, 0);
Åsa Perssonc29cb2c2019-03-25 11:06:59543
544 video_source_.IncomingCapturedFrame(
545 CreateFrame(1, codec_width_, codec_height_));
546 WaitForEncodedFrame(1);
547 }
548
asapersson02465b82017-04-10 08:12:52549 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 08:12:52550 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-16 06:40:18551 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
552 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 08:12:52553 }
554
asapersson09f05612017-05-16 06:40:18555 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
556 const rtc::VideoSinkWants& wants2) {
557 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
558 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
559 }
560
Åsa Persson8c1bf952018-09-13 08:42:19561 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
562 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
563 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
564 EXPECT_FALSE(wants.target_pixel_count);
565 }
566
asapersson09f05612017-05-16 06:40:18567 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
568 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 08:42:19569 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-16 06:40:18570 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
571 EXPECT_GT(wants1.max_pixel_count, 0);
572 }
573
574 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
575 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 08:42:19576 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-16 06:40:18577 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
578 }
579
asaperssonf7e294d2017-06-14 06:25:22580 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
581 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 08:42:19582 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:22583 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
584 }
585
586 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
587 const rtc::VideoSinkWants& wants2) {
588 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
589 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
590 }
591
592 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
593 const rtc::VideoSinkWants& wants2) {
594 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
595 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
596 }
597
598 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
599 const rtc::VideoSinkWants& wants2) {
600 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
601 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
602 EXPECT_GT(wants1.max_pixel_count, 0);
603 }
604
605 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
606 const rtc::VideoSinkWants& wants2) {
607 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
608 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
609 }
610
asapersson09f05612017-05-16 06:40:18611 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
612 int pixel_count) {
Åsa Persson8c1bf952018-09-13 08:42:19613 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 08:12:52614 EXPECT_LT(wants.max_pixel_count, pixel_count);
615 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-16 06:40:18616 }
617
618 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
619 EXPECT_LT(wants.max_framerate_fps, fps);
620 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
621 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 08:12:52622 }
623
asaperssonf7e294d2017-06-14 06:25:22624 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
625 int expected_fps) {
626 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
627 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
628 EXPECT_FALSE(wants.target_pixel_count);
629 }
630
Jonathan Yubc771b72017-12-09 01:04:29631 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
632 int last_frame_pixels) {
633 // Balanced mode should always scale FPS to the desired range before
634 // attempting to scale resolution.
635 int fps_limit = wants.max_framerate_fps;
636 if (last_frame_pixels <= 320 * 240) {
637 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
638 } else if (last_frame_pixels <= 480 * 270) {
639 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
640 } else if (last_frame_pixels <= 640 * 480) {
641 EXPECT_LE(15, fps_limit);
642 } else {
Åsa Persson8c1bf952018-09-13 08:42:19643 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-09 01:04:29644 }
645 }
646
sprang4847ae62017-06-27 14:06:52647 void WaitForEncodedFrame(int64_t expected_ntp_time) {
648 sink_.WaitForEncodedFrame(expected_ntp_time);
Sebastian Jansson40889f32019-04-17 10:11:20649 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52650 }
651
652 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
653 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Sebastian Jansson40889f32019-04-17 10:11:20654 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52655 return ok;
656 }
657
658 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
659 sink_.WaitForEncodedFrame(expected_width, expected_height);
Sebastian Jansson40889f32019-04-17 10:11:20660 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52661 }
662
663 void ExpectDroppedFrame() {
664 sink_.ExpectDroppedFrame();
Sebastian Jansson40889f32019-04-17 10:11:20665 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52666 }
667
668 bool WaitForFrame(int64_t timeout_ms) {
669 bool ok = sink_.WaitForFrame(timeout_ms);
Sebastian Jansson40889f32019-04-17 10:11:20670 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 14:06:52671 return ok;
672 }
673
perkj26091b12016-09-01 08:17:40674 class TestEncoder : public test::FakeEncoder {
675 public:
Niels Möllerc572ff32018-11-07 07:43:50676 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 08:17:40677
asaperssonfab67072017-04-04 12:51:49678 VideoCodec codec_config() const {
brandtre78d2662017-01-16 13:57:16679 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-03 06:45:26680 return config_;
681 }
682
683 void BlockNextEncode() {
brandtre78d2662017-01-16 13:57:16684 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-03 06:45:26685 block_next_encode_ = true;
686 }
687
Erik Språngaed30702018-11-05 11:57:17688 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 08:24:41689 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 11:42:18690 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 14:52:33691 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 11:42:18692 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 14:56:00693 info.scaling_settings = VideoEncoder::ScalingSettings(
694 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 11:42:18695 }
696 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 11:06:59697 for (int i = 0; i < kMaxSpatialLayers; ++i) {
698 if (temporal_layers_supported_[i]) {
699 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
700 info.fps_allocation[i].resize(num_layers);
701 }
702 }
Erik Språngaed30702018-11-05 11:57:17703 }
Sergey Silkin6456e352019-07-08 15:56:40704
705 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Erik Språngaed30702018-11-05 11:57:17706 return info;
kthelgason876222f2016-11-29 09:44:11707 }
708
Erik Språngb7cb7b52019-02-26 14:52:33709 int32_t RegisterEncodeCompleteCallback(
710 EncodedImageCallback* callback) override {
711 rtc::CritScope lock(&local_crit_sect_);
712 encoded_image_callback_ = callback;
713 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
714 }
715
perkjfa10b552016-10-03 06:45:26716 void ContinueEncode() { continue_encode_event_.Set(); }
717
718 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
719 uint32_t timestamp) const {
brandtre78d2662017-01-16 13:57:16720 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-03 06:45:26721 EXPECT_EQ(timestamp_, timestamp);
722 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
723 }
724
kthelgason2fc52542017-03-03 08:24:41725 void SetQualityScaling(bool b) {
726 rtc::CritScope lock(&local_crit_sect_);
727 quality_scaling_ = b;
728 }
kthelgasonad9010c2017-02-14 08:46:51729
Mirta Dvornicicccc1b572019-01-15 11:42:18730 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
731 rtc::CritScope lock(&local_crit_sect_);
732 is_hardware_accelerated_ = is_hardware_accelerated;
733 }
734
Åsa Perssonc29cb2c2019-03-25 11:06:59735 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
736 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
737 rtc::CritScope lock(&local_crit_sect_);
738 temporal_layers_supported_[spatial_idx] = supported;
739 }
740
Sergey Silkin6456e352019-07-08 15:56:40741 void SetResolutionBitrateLimits(
742 std::vector<ResolutionBitrateLimits> thresholds) {
743 rtc::CritScope cs(&local_crit_sect_);
744 resolution_bitrate_limits_ = thresholds;
745 }
746
sprangfe627f32017-03-29 15:24:59747 void ForceInitEncodeFailure(bool force_failure) {
748 rtc::CritScope lock(&local_crit_sect_);
749 force_init_encode_failed_ = force_failure;
750 }
751
Niels Möller6bb5ab92019-01-11 10:11:10752 void SimulateOvershoot(double rate_factor) {
753 rtc::CritScope lock(&local_crit_sect_);
754 rate_factor_ = rate_factor;
755 }
756
Erik Språngd7329ca2019-02-21 20:19:53757 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 10:11:10758 rtc::CritScope lock(&local_crit_sect_);
759 return last_framerate_;
760 }
761
Erik Språngd7329ca2019-02-21 20:19:53762 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26763 rtc::CritScope lock(&local_crit_sect_);
764 return last_update_rect_;
765 }
766
Niels Möller87e2d782019-03-07 09:18:23767 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 20:19:53768 rtc::CritScope lock(&local_crit_sect_);
769 return last_frame_types_;
770 }
771
772 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 09:18:23773 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 14:43:58774 keyframe ? VideoFrameType::kVideoFrameKey
775 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:53776 {
777 rtc::CritScope lock(&local_crit_sect_);
778 last_frame_types_ = frame_type;
779 }
Niels Möllerb859b322019-03-07 11:40:01780 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 20:19:53781 }
782
Erik Språngb7cb7b52019-02-26 14:52:33783 void InjectEncodedImage(const EncodedImage& image) {
784 rtc::CritScope lock(&local_crit_sect_);
785 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
786 }
787
Mirta Dvornicic28f0eb22019-05-28 14:30:16788 void InjectEncodedImage(const EncodedImage& image,
789 const CodecSpecificInfo* codec_specific_info,
790 const RTPFragmentationHeader* fragmentation) {
791 rtc::CritScope lock(&local_crit_sect_);
792 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
793 fragmentation);
794 }
795
Erik Språngd7329ca2019-02-21 20:19:53796 void ExpectNullFrame() {
797 rtc::CritScope lock(&local_crit_sect_);
798 expect_null_frame_ = true;
799 }
800
Erik Språng5056af02019-09-02 13:53:11801 absl::optional<VideoEncoder::RateControlParameters>
802 GetAndResetLastRateControlSettings() {
803 auto settings = last_rate_control_settings_;
804 last_rate_control_settings_.reset();
805 return settings;
Erik Språngd7329ca2019-02-21 20:19:53806 }
807
Sergey Silkin5ee69672019-07-02 12:18:34808 int GetNumEncoderInitializations() const {
809 rtc::CritScope lock(&local_crit_sect_);
810 return num_encoder_initializations_;
811 }
812
Evan Shrubsole7c079f62019-09-26 07:55:03813 int GetNumSetRates() const {
814 rtc::CritScope lock(&local_crit_sect_);
815 return num_set_rates_;
816 }
817
perkjfa10b552016-10-03 06:45:26818 private:
perkj26091b12016-09-01 08:17:40819 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 09:18:23820 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 08:17:40821 bool block_encode;
822 {
brandtre78d2662017-01-16 13:57:16823 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 20:19:53824 if (expect_null_frame_) {
825 EXPECT_EQ(input_image.timestamp(), 0u);
826 EXPECT_EQ(input_image.width(), 1);
827 last_frame_types_ = *frame_types;
828 expect_null_frame_ = false;
829 } else {
830 EXPECT_GT(input_image.timestamp(), timestamp_);
831 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
832 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
833 }
perkj26091b12016-09-01 08:17:40834
835 timestamp_ = input_image.timestamp();
836 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 18:45:46837 last_input_width_ = input_image.width();
838 last_input_height_ = input_image.height();
perkj26091b12016-09-01 08:17:40839 block_encode = block_next_encode_;
840 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26841 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 20:19:53842 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 08:17:40843 }
Niels Möllerb859b322019-03-07 11:40:01844 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 08:17:40845 if (block_encode)
perkja49cbd32016-09-16 14:53:41846 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:40847 return result;
848 }
849
sprangfe627f32017-03-29 15:24:59850 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 12:57:57851 const Settings& settings) override {
852 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 12:18:34853
sprangfe627f32017-03-29 15:24:59854 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 14:52:33855 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 12:18:34856
857 ++num_encoder_initializations_;
858
Erik Språng82fad3d2018-03-21 08:57:23859 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 15:24:59860 // Simulate setting up temporal layers, in order to validate the life
861 // cycle of these objects.
Elad Aloncde8ab22019-03-20 10:56:20862 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 09:20:09863 frame_buffer_controller_ =
864 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 15:24:59865 }
Erik Språngb7cb7b52019-02-26 14:52:33866 if (force_init_encode_failed_) {
867 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 15:24:59868 return -1;
Erik Språngb7cb7b52019-02-26 14:52:33869 }
Mirta Dvornicicccc1b572019-01-15 11:42:18870
Erik Språngb7cb7b52019-02-26 14:52:33871 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 15:24:59872 return res;
873 }
874
Erik Språngb7cb7b52019-02-26 14:52:33875 int32_t Release() override {
876 rtc::CritScope lock(&local_crit_sect_);
877 EXPECT_NE(initialized_, EncoderState::kUninitialized);
878 initialized_ = EncoderState::kUninitialized;
879 return FakeEncoder::Release();
880 }
881
Erik Språng16cb8f52019-04-12 11:59:09882 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 10:11:10883 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 07:55:03884 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 10:11:10885 VideoBitrateAllocation adjusted_rate_allocation;
886 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
887 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 11:59:09888 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 10:11:10889 adjusted_rate_allocation.SetBitrate(
890 si, ti,
Erik Språng16cb8f52019-04-12 11:59:09891 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 10:11:10892 rate_factor_));
893 }
894 }
895 }
Erik Språng16cb8f52019-04-12 11:59:09896 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 13:53:11897 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 11:59:09898 RateControlParameters adjusted_paramters = parameters;
899 adjusted_paramters.bitrate = adjusted_rate_allocation;
900 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 10:11:10901 }
902
brandtre78d2662017-01-16 13:57:16903 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 14:52:33904 enum class EncoderState {
905 kUninitialized,
906 kInitializationFailed,
907 kInitialized
908 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
909 EncoderState::kUninitialized;
danilchapa37de392017-09-09 11:17:22910 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 08:17:40911 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 11:17:22912 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
913 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
914 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
915 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
916 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 11:42:18917 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 10:56:20918 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 11:17:22919 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 11:06:59920 absl::optional<bool>
921 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
922 local_crit_sect_);
danilchapa37de392017-09-09 11:17:22923 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 10:11:10924 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
925 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 13:53:11926 absl::optional<VideoEncoder::RateControlParameters>
927 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:26928 VideoFrame::UpdateRect last_update_rect_
929 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 09:18:23930 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 20:19:53931 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 14:52:33932 EncodedImageCallback* encoded_image_callback_
933 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Elad Alon45befc52019-07-02 09:20:09934 MockFecControllerOverride fec_controller_override_;
Sergey Silkin5ee69672019-07-02 12:18:34935 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 15:56:40936 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
937 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 07:55:03938 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 08:17:40939 };
940
mflodmancc3d4422017-08-03 15:27:51941 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 08:17:40942 public:
943 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 07:43:50944 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 08:17:40945
perkj26091b12016-09-01 08:17:40946 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 14:06:52947 EXPECT_TRUE(
948 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
949 }
950
951 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
952 int64_t timeout_ms) {
perkj26091b12016-09-01 08:17:40953 uint32_t timestamp = 0;
sprang4847ae62017-06-27 14:06:52954 if (!encoded_frame_event_.Wait(timeout_ms))
955 return false;
perkj26091b12016-09-01 08:17:40956 {
957 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 16:38:12958 timestamp = last_timestamp_;
perkj26091b12016-09-01 08:17:40959 }
960 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 14:06:52961 return true;
perkj26091b12016-09-01 08:17:40962 }
963
sprangb1ca0732017-02-01 16:38:12964 void WaitForEncodedFrame(uint32_t expected_width,
965 uint32_t expected_height) {
sprangc5d62e22017-04-03 06:53:04966 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 13:13:56967 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-03 06:53:04968 }
969
Åsa Perssonc74d8da2017-12-04 13:13:56970 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-03 06:53:04971 uint32_t expected_height) {
sprangb1ca0732017-02-01 16:38:12972 uint32_t width = 0;
973 uint32_t height = 0;
sprangb1ca0732017-02-01 16:38:12974 {
975 rtc::CritScope lock(&crit_);
976 width = last_width_;
977 height = last_height_;
978 }
979 EXPECT_EQ(expected_height, height);
980 EXPECT_EQ(expected_width, width);
981 }
982
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:14983 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
984 VideoRotation rotation;
985 {
986 rtc::CritScope lock(&crit_);
987 rotation = last_rotation_;
988 }
989 EXPECT_EQ(expected_rotation, rotation);
990 }
991
kthelgason2fc52542017-03-03 08:24:41992 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 15:02:22993
sprangc5d62e22017-04-03 06:53:04994 bool WaitForFrame(int64_t timeout_ms) {
995 return encoded_frame_event_.Wait(timeout_ms);
996 }
997
perkj26091b12016-09-01 08:17:40998 void SetExpectNoFrames() {
999 rtc::CritScope lock(&crit_);
1000 expect_frames_ = false;
1001 }
1002
asaperssonfab67072017-04-04 12:51:491003 int number_of_reconfigurations() const {
Per512ecb32016-09-23 13:52:061004 rtc::CritScope lock(&crit_);
1005 return number_of_reconfigurations_;
1006 }
1007
asaperssonfab67072017-04-04 12:51:491008 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 13:52:061009 rtc::CritScope lock(&crit_);
1010 return min_transmit_bitrate_bps_;
1011 }
1012
Erik Språngd7329ca2019-02-21 20:19:531013 void SetNumExpectedLayers(size_t num_layers) {
1014 rtc::CritScope lock(&crit_);
1015 num_expected_layers_ = num_layers;
1016 }
1017
Erik Språngb7cb7b52019-02-26 14:52:331018 int64_t GetLastCaptureTimeMs() const {
1019 rtc::CritScope lock(&crit_);
1020 return last_capture_time_ms_;
1021 }
1022
Mirta Dvornicic28f0eb22019-05-28 14:30:161023 std::vector<uint8_t> GetLastEncodedImageData() {
1024 rtc::CritScope lock(&crit_);
1025 return std::move(last_encoded_image_data_);
1026 }
1027
1028 RTPFragmentationHeader GetLastFragmentation() {
1029 rtc::CritScope lock(&crit_);
1030 return std::move(last_fragmentation_);
1031 }
1032
perkj26091b12016-09-01 08:17:401033 private:
sergeyu2cb155a2016-11-04 18:39:291034 Result OnEncodedImage(
1035 const EncodedImage& encoded_image,
1036 const CodecSpecificInfo* codec_specific_info,
1037 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 13:52:061038 rtc::CritScope lock(&crit_);
1039 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 14:30:161040 last_encoded_image_data_ = std::vector<uint8_t>(
1041 encoded_image.data(), encoded_image.data() + encoded_image.size());
1042 if (fragmentation) {
1043 last_fragmentation_.CopyFrom(*fragmentation);
1044 }
Erik Språngd7329ca2019-02-21 20:19:531045 uint32_t timestamp = encoded_image.Timestamp();
1046 if (last_timestamp_ != timestamp) {
1047 num_received_layers_ = 1;
1048 } else {
1049 ++num_received_layers_;
1050 }
1051 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 14:52:331052 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 16:38:121053 last_width_ = encoded_image._encodedWidth;
1054 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141055 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 20:19:531056 if (num_received_layers_ == num_expected_layers_) {
1057 encoded_frame_event_.Set();
1058 }
sprangb1ca0732017-02-01 16:38:121059 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 13:52:061060 }
1061
Rasmus Brandtc402dbe2019-02-04 10:09:461062 void OnEncoderConfigurationChanged(
1063 std::vector<VideoStream> streams,
1064 VideoEncoderConfig::ContentType content_type,
1065 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 12:18:341066 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 13:52:061067 ++number_of_reconfigurations_;
1068 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1069 }
1070
perkj26091b12016-09-01 08:17:401071 rtc::CriticalSection crit_;
1072 TestEncoder* test_encoder_;
1073 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 14:30:161074 std::vector<uint8_t> last_encoded_image_data_;
1075 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 16:38:121076 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 14:52:331077 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 16:38:121078 uint32_t last_height_ = 0;
1079 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:141080 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 20:19:531081 size_t num_expected_layers_ = 1;
1082 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 08:17:401083 bool expect_frames_ = true;
Per512ecb32016-09-23 13:52:061084 int number_of_reconfigurations_ = 0;
1085 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 08:17:401086 };
1087
Sergey Silkin5ee69672019-07-02 12:18:341088 class VideoBitrateAllocatorProxyFactory
1089 : public VideoBitrateAllocatorFactory {
1090 public:
1091 VideoBitrateAllocatorProxyFactory()
1092 : bitrate_allocator_factory_(
1093 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1094
1095 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1096 const VideoCodec& codec) override {
1097 rtc::CritScope lock(&crit_);
1098 codec_config_ = codec;
1099 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1100 }
1101
1102 VideoCodec codec_config() const {
1103 rtc::CritScope lock(&crit_);
1104 return codec_config_;
1105 }
1106
1107 private:
1108 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1109
1110 rtc::CriticalSection crit_;
1111 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1112 };
1113
perkj26091b12016-09-01 08:17:401114 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 15:41:301115 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 13:52:061116 int codec_width_;
1117 int codec_height_;
sprang4847ae62017-06-27 14:06:521118 int max_framerate_;
Erik Språng82268752019-08-29 13:07:471119 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 15:01:231120 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 08:17:401121 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 07:07:241122 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 12:18:341123 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-03 06:53:041124 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 08:17:401125 TestSink sink_;
sprangb1ca0732017-02-01 16:38:121126 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 15:27:511127 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 08:17:401128};
1129
mflodmancc3d4422017-08-03 15:27:511130TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 13:14:011131 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551132 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1133 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerc572ff32018-11-07 07:43:501134 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411135 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 14:06:521136 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 14:53:411137 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 15:27:511138 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401139}
1140
mflodmancc3d4422017-08-03 15:27:511141TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 08:17:401142 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 07:43:501143 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 11:05:491144 // The encoder will cache up to one frame for a short duration. Adding two
1145 // frames means that the first frame will be dropped and the second frame will
1146 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 14:53:411147 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 11:05:491148 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 14:53:411149 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401150
Erik Språng4c6ca302019-04-08 13:14:011151 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551152 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1153 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 08:17:401154
Sebastian Janssona3177052018-04-10 11:05:491155 // The pending frame should be received.
sprang4847ae62017-06-27 14:06:521156 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 11:05:491157 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1158
1159 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:511160 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401161}
1162
mflodmancc3d4422017-08-03 15:27:511163TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 13:14:011164 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551165 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1166 DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 14:53:411167 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521168 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401169
Florent Castellia8336d32019-09-09 11:36:551170 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0),
1171 DataRate::bps(0), 0, 0);
Sebastian Janssona3177052018-04-10 11:05:491172 // The encoder will cache up to one frame for a short duration. Adding two
1173 // frames means that the first frame will be dropped and the second frame will
1174 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 14:53:411175 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 11:05:491176 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401177
Erik Språng4c6ca302019-04-08 13:14:011178 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551179 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1180 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 14:06:521181 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 11:05:491182 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1183 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 15:27:511184 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401185}
1186
mflodmancc3d4422017-08-03 15:27:511187TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 13:14:011188 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551189 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1190 DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 14:53:411191 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521192 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401193
1194 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 14:53:411195 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 08:17:401196
perkja49cbd32016-09-16 14:53:411197 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521198 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:511199 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401200}
1201
mflodmancc3d4422017-08-03 15:27:511202TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 13:14:011203 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551204 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1205 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 08:17:401206
perkja49cbd32016-09-16 14:53:411207 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521208 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401209
mflodmancc3d4422017-08-03 15:27:511210 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401211 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 07:43:501212 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 14:53:411213 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1214 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 08:17:401215}
1216
mflodmancc3d4422017-08-03 15:27:511217TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 13:14:011218 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551219 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1220 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 08:17:401221
1222 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 14:53:411223 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521224 WaitForEncodedFrame(1);
perkj26091b12016-09-01 08:17:401225 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1226 // call to ContinueEncode.
perkja49cbd32016-09-16 14:53:411227 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1228 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 08:17:401229 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 14:06:521230 WaitForEncodedFrame(3);
perkj26091b12016-09-01 08:17:401231
mflodmancc3d4422017-08-03 15:27:511232 video_stream_encoder_->Stop();
perkj26091b12016-09-01 08:17:401233}
1234
Noah Richards51db4212019-06-12 13:59:121235TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1236 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551237 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1238 DataRate::bps(kTargetBitrateBps), 0, 0);
Noah Richards51db4212019-06-12 13:59:121239
1240 rtc::Event frame_destroyed_event;
1241 video_source_.IncomingCapturedFrame(
1242 CreateFakeNativeFrame(1, &frame_destroyed_event));
1243 ExpectDroppedFrame();
1244 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1245 video_stream_encoder_->Stop();
1246}
1247
1248TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1249 // Use the cropping factory.
1250 video_encoder_config_.video_stream_factory =
1251 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1252 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1253 kMaxPayloadLength);
1254 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1255
1256 // Capture a frame at codec_width_/codec_height_.
1257 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551258 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1259 DataRate::bps(kTargetBitrateBps), 0, 0);
Noah Richards51db4212019-06-12 13:59:121260 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1261 WaitForEncodedFrame(1);
1262 // The encoder will have been configured once.
1263 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1264 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1265 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1266
1267 // Now send in a fake frame that needs to be cropped as the width/height
1268 // aren't divisible by 4 (see CreateEncoderStreams above).
1269 rtc::Event frame_destroyed_event;
1270 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1271 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1272 ExpectDroppedFrame();
1273 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1274 video_stream_encoder_->Stop();
1275}
1276
mflodmancc3d4422017-08-03 15:27:511277TEST_F(VideoStreamEncoderTest,
1278 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 13:14:011279 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551280 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1281 DataRate::bps(kTargetBitrateBps), 0, 0);
Per21d45d22016-10-30 20:37:571282 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061283
1284 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551285 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521286 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571287 // The encoder will have been configured once when the first frame is
1288 // received.
1289 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 13:52:061290
1291 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:511292 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 13:52:061293 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 15:27:511294 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:471295 kMaxPayloadLength);
Per512ecb32016-09-23 13:52:061296
1297 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 14:24:551298 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521299 WaitForEncodedFrame(2);
Per21d45d22016-10-30 20:37:571300 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-30 06:25:401301 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-30 05:39:101302
mflodmancc3d4422017-08-03 15:27:511303 video_stream_encoder_->Stop();
perkj26105b42016-09-30 05:39:101304}
1305
mflodmancc3d4422017-08-03 15:27:511306TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 13:14:011307 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551308 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1309 DataRate::bps(kTargetBitrateBps), 0, 0);
perkjfa10b552016-10-03 06:45:261310
1311 // Capture a frame and wait for it to synchronize with the encoder thread.
1312 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 14:06:521313 WaitForEncodedFrame(1);
Per21d45d22016-10-30 20:37:571314 // The encoder will have been configured once.
1315 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261316 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1317 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1318
1319 codec_width_ *= 2;
1320 codec_height_ *= 2;
1321 // Capture a frame with a higher resolution and wait for it to synchronize
1322 // with the encoder thread.
1323 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 14:06:521324 WaitForEncodedFrame(2);
perkjfa10b552016-10-03 06:45:261325 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1326 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 20:37:571327 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-03 06:45:261328
mflodmancc3d4422017-08-03 15:27:511329 video_stream_encoder_->Stop();
perkjfa10b552016-10-03 06:45:261330}
1331
Sergey Silkin443b7ee2019-06-28 10:53:071332TEST_F(VideoStreamEncoderTest,
1333 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1334 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551335 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1336 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin443b7ee2019-06-28 10:53:071337
1338 // Capture a frame and wait for it to synchronize with the encoder thread.
1339 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1340 WaitForEncodedFrame(1);
1341
1342 VideoEncoderConfig video_encoder_config;
1343 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1344 // Changing the max payload data length recreates encoder.
1345 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1346 kMaxPayloadLength / 2);
1347
1348 // Capture a frame and wait for it to synchronize with the encoder thread.
1349 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1350 WaitForEncodedFrame(2);
1351 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1352
1353 video_stream_encoder_->Stop();
1354}
1355
Sergey Silkin5ee69672019-07-02 12:18:341356TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1357 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551358 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1359 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin5ee69672019-07-02 12:18:341360
1361 VideoEncoderConfig video_encoder_config;
1362 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1363 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1364 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1365 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1366 kMaxPayloadLength);
1367
1368 // Capture a frame and wait for it to synchronize with the encoder thread.
1369 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1370 WaitForEncodedFrame(1);
1371 // The encoder will have been configured once when the first frame is
1372 // received.
1373 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1374 EXPECT_EQ(kTargetBitrateBps,
1375 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1376 EXPECT_EQ(kStartBitrateBps,
1377 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1378
Sergey Silkin6456e352019-07-08 15:56:401379 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1380 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 12:18:341381 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1382 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1383 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1384 kMaxPayloadLength);
1385
1386 // Capture a frame and wait for it to synchronize with the encoder thread.
1387 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1388 WaitForEncodedFrame(2);
1389 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1390 // Bitrate limits have changed - rate allocator should be reconfigured,
1391 // encoder should not be reconfigured.
1392 EXPECT_EQ(kTargetBitrateBps * 2,
1393 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1394 EXPECT_EQ(kStartBitrateBps * 2,
1395 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1396 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1397
1398 video_stream_encoder_->Stop();
1399}
1400
Sergey Silkin6456e352019-07-08 15:56:401401TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 14:04:051402 EncoderRecommendedBitrateLimitsDoNotOverrideAppBitrateLimits) {
Sergey Silkin6456e352019-07-08 15:56:401403 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551404 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1405 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401406
Sergey Silkin6456e352019-07-08 15:56:401407 VideoEncoderConfig video_encoder_config;
1408 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1409 video_encoder_config.max_bitrate_bps = 0;
Sergey Silkin6b2cec12019-08-09 14:04:051410 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
Sergey Silkin6456e352019-07-08 15:56:401411 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1412 kMaxPayloadLength);
1413
Sergey Silkin6b2cec12019-08-09 14:04:051414 video_source_.IncomingCapturedFrame(CreateFrame(1, 360, 180));
Sergey Silkin6456e352019-07-08 15:56:401415 WaitForEncodedFrame(1);
Sergey Silkin6456e352019-07-08 15:56:401416
Sergey Silkin6b2cec12019-08-09 14:04:051417 // Get the default bitrate limits and use them as baseline for custom
1418 // application and encoder recommended limits.
1419 const uint32_t kDefaultMinBitrateKbps =
1420 bitrate_allocator_factory_.codec_config().minBitrate;
1421 const uint32_t kDefaultMaxBitrateKbps =
1422 bitrate_allocator_factory_.codec_config().maxBitrate;
1423 const uint32_t kEncMinBitrateKbps = kDefaultMinBitrateKbps * 2;
1424 const uint32_t kEncMaxBitrateKbps = kDefaultMaxBitrateKbps * 2;
1425 const uint32_t kAppMinBitrateKbps = kDefaultMinBitrateKbps * 3;
1426 const uint32_t kAppMaxBitrateKbps = kDefaultMaxBitrateKbps * 3;
1427
1428 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1429 codec_width_ * codec_height_, kEncMinBitrateKbps * 1000,
1430 kEncMinBitrateKbps * 1000, kEncMaxBitrateKbps * 1000);
1431 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1432
1433 // Change resolution. This will trigger encoder re-configuration and video
1434 // stream encoder will pick up the bitrate limits recommended by encoder.
1435 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1436 WaitForEncodedFrame(2);
1437 video_source_.IncomingCapturedFrame(CreateFrame(3, 360, 180));
1438 WaitForEncodedFrame(3);
1439
1440 // App bitrate limits are not set - bitrate limits recommended by encoder
1441 // should be used.
1442 EXPECT_EQ(kEncMaxBitrateKbps,
1443 bitrate_allocator_factory_.codec_config().maxBitrate);
1444 EXPECT_EQ(kEncMinBitrateKbps,
1445 bitrate_allocator_factory_.codec_config().minBitrate);
1446
1447 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1448 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
1449 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1450 kMaxPayloadLength);
1451 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1452 WaitForEncodedFrame(4);
1453
1454 // App limited the max bitrate - bitrate limits recommended by encoder should
1455 // not be applied.
1456 EXPECT_EQ(kAppMaxBitrateKbps,
1457 bitrate_allocator_factory_.codec_config().maxBitrate);
1458 EXPECT_EQ(kDefaultMinBitrateKbps,
1459 bitrate_allocator_factory_.codec_config().minBitrate);
1460
1461 video_encoder_config.max_bitrate_bps = 0;
1462 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1463 kAppMinBitrateKbps * 1000;
1464 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1465 kMaxPayloadLength);
1466 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1467 WaitForEncodedFrame(5);
1468
1469 // App limited the min bitrate - bitrate limits recommended by encoder should
1470 // not be applied.
1471 EXPECT_EQ(kDefaultMaxBitrateKbps,
1472 bitrate_allocator_factory_.codec_config().maxBitrate);
1473 EXPECT_EQ(kAppMinBitrateKbps,
1474 bitrate_allocator_factory_.codec_config().minBitrate);
1475
1476 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1477 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1478 kAppMinBitrateKbps * 1000;
Sergey Silkin6456e352019-07-08 15:56:401479 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1480 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 14:04:051481 video_source_.IncomingCapturedFrame(CreateFrame(6, nullptr));
1482 WaitForEncodedFrame(6);
Sergey Silkin6456e352019-07-08 15:56:401483
Sergey Silkin6b2cec12019-08-09 14:04:051484 // App limited both min and max bitrates - bitrate limits recommended by
1485 // encoder should not be applied.
1486 EXPECT_EQ(kAppMaxBitrateKbps,
1487 bitrate_allocator_factory_.codec_config().maxBitrate);
1488 EXPECT_EQ(kAppMinBitrateKbps,
1489 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 15:56:401490
1491 video_stream_encoder_->Stop();
1492}
1493
1494TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 14:04:051495 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 15:56:401496 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551497 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1498 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6456e352019-07-08 15:56:401499
1500 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 14:04:051501 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 15:56:401502 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 14:04:051503 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 15:56:401504 fake_encoder_.SetResolutionBitrateLimits(
1505 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1506
1507 VideoEncoderConfig video_encoder_config;
1508 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1509 video_encoder_config.max_bitrate_bps = 0;
1510 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1511 kMaxPayloadLength);
1512
Sergey Silkin6b2cec12019-08-09 14:04:051513 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 15:56:401514 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1515 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 14:04:051516 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1517 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401518 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1519 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1520
Sergey Silkin6b2cec12019-08-09 14:04:051521 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 15:56:401522 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1523 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 14:04:051524 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1525 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401526 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1527 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1528
Sergey Silkin6b2cec12019-08-09 14:04:051529 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 15:56:401530 // encoder for 360p should be used.
1531 video_source_.IncomingCapturedFrame(
1532 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1533 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 14:04:051534 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1535 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401536 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1537 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1538
Sergey Silkin6b2cec12019-08-09 14:04:051539 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 15:56:401540 // ignored.
1541 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1542 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 14:04:051543 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1544 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401545 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1546 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 14:04:051547 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1548 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401549 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1550 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1551
1552 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1553 // for 270p should be used.
1554 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1555 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 14:04:051556 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1557 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 15:56:401558 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1559 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1560
1561 video_stream_encoder_->Stop();
1562}
1563
Sergey Silkin6b2cec12019-08-09 14:04:051564TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1565 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551566 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1567 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6b2cec12019-08-09 14:04:051568
1569 VideoEncoderConfig video_encoder_config;
1570 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1571 video_encoder_config.max_bitrate_bps = 0;
1572 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1573 kMaxPayloadLength);
1574
1575 // Encode 720p frame to get the default encoder target bitrate.
1576 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1577 WaitForEncodedFrame(1);
1578 const uint32_t kDefaultTargetBitrateFor720pKbps =
1579 bitrate_allocator_factory_.codec_config()
1580 .simulcastStream[0]
1581 .targetBitrate;
1582
1583 // Set the max recommended encoder bitrate to something lower than the default
1584 // target bitrate.
1585 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1586 1280 * 720, 10 * 1000, 10 * 1000,
1587 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1588 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1589
1590 // Change resolution to trigger encoder reinitialization.
1591 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1592 WaitForEncodedFrame(2);
1593 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1594 WaitForEncodedFrame(3);
1595
1596 // Ensure the target bitrate is capped by the max bitrate.
1597 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1598 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1599 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1600 .simulcastStream[0]
1601 .targetBitrate *
1602 1000,
1603 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1604
1605 video_stream_encoder_->Stop();
1606}
1607
mflodmancc3d4422017-08-03 15:27:511608TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 18:45:461609 EXPECT_TRUE(video_source_.has_sinks());
1610 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:511611 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411612 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:461613 EXPECT_FALSE(video_source_.has_sinks());
1614 EXPECT_TRUE(new_video_source.has_sinks());
1615
mflodmancc3d4422017-08-03 15:27:511616 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:461617}
1618
mflodmancc3d4422017-08-03 15:27:511619TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 18:45:461620 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:511621 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 18:45:461622 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 15:27:511623 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:461624}
1625
Jonathan Yubc771b72017-12-09 01:04:291626TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1627 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-14 06:25:221628 const int kWidth = 1280;
1629 const int kHeight = 720;
Jonathan Yubc771b72017-12-09 01:04:291630
1631 // We rely on the automatic resolution adaptation, but we handle framerate
1632 // adaptation manually by mocking the stats proxy.
1633 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-14 06:25:221634
Taylor Brandstetter49fcc102018-05-16 21:20:411635 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 13:14:011636 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551637 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1638 DataRate::bps(kTargetBitrateBps), 0, 0);
Taylor Brandstetter49fcc102018-05-16 21:20:411639 video_stream_encoder_->SetSource(&video_source_,
1640 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-09 01:04:291641 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-14 06:25:221642 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:291643 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-14 06:25:221644 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1645
Jonathan Yubc771b72017-12-09 01:04:291646 // Adapt down as far as possible.
1647 rtc::VideoSinkWants last_wants;
1648 int64_t t = 1;
1649 int loop_count = 0;
1650 do {
1651 ++loop_count;
1652 last_wants = video_source_.sink_wants();
1653
1654 // Simulate the framerate we've been asked to adapt to.
1655 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1656 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1657 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1658 mock_stats.input_frame_rate = fps;
1659 stats_proxy_->SetMockStats(mock_stats);
1660
1661 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1662 sink_.WaitForEncodedFrame(t);
1663 t += frame_interval_ms;
1664
mflodmancc3d4422017-08-03 15:27:511665 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:291666 VerifyBalancedModeFpsRange(
1667 video_source_.sink_wants(),
1668 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1669 } while (video_source_.sink_wants().max_pixel_count <
1670 last_wants.max_pixel_count ||
1671 video_source_.sink_wants().max_framerate_fps <
1672 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:221673
Jonathan Yubc771b72017-12-09 01:04:291674 // Verify that we've adapted all the way down.
1675 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:221676 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:291677 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1678 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-14 06:25:221679 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-09 01:04:291680 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1681 *video_source_.last_sent_height());
1682 EXPECT_EQ(kMinBalancedFramerateFps,
1683 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:221684
Jonathan Yubc771b72017-12-09 01:04:291685 // Adapt back up the same number of times we adapted down.
1686 for (int i = 0; i < loop_count - 1; ++i) {
1687 last_wants = video_source_.sink_wants();
1688
1689 // Simulate the framerate we've been asked to adapt to.
1690 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1691 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1692 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1693 mock_stats.input_frame_rate = fps;
1694 stats_proxy_->SetMockStats(mock_stats);
1695
1696 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1697 sink_.WaitForEncodedFrame(t);
1698 t += frame_interval_ms;
1699
mflodmancc3d4422017-08-03 15:27:511700 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-09 01:04:291701 VerifyBalancedModeFpsRange(
1702 video_source_.sink_wants(),
1703 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1704 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1705 last_wants.max_pixel_count ||
1706 video_source_.sink_wants().max_framerate_fps >
1707 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-14 06:25:221708 }
1709
Åsa Persson8c1bf952018-09-13 08:42:191710 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-09 01:04:291711 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-14 06:25:221712 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:291713 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1714 EXPECT_EQ((loop_count - 1) * 2,
1715 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-14 06:25:221716
mflodmancc3d4422017-08-03 15:27:511717 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:221718}
mflodmancc3d4422017-08-03 15:27:511719TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 13:14:011720 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551721 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1722 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 08:12:521723 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 18:45:461724
sprangc5d62e22017-04-03 06:53:041725 const int kFrameWidth = 1280;
1726 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:041727
Åsa Persson8c1bf952018-09-13 08:42:191728 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 18:45:461729
kthelgason5e13d412016-12-01 11:59:511730 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:041731 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:521732 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:041733 frame_timestamp += kFrameIntervalMs;
1734
perkj803d97f2016-11-01 18:45:461735 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:511736 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:461737 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:041738 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:521739 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:041740 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 14:23:481741
asapersson0944a802017-04-07 07:57:581742 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-03 06:53:041743 // wanted resolution.
1744 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1745 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1746 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 08:42:191747 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:041748
1749 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 18:45:461750 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:511751 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411752 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 18:45:461753
sprangc5d62e22017-04-03 06:53:041754 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 08:42:191755 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 18:45:461756
sprangc5d62e22017-04-03 06:53:041757 // Force an input frame rate to be available, or the adaptation call won't
1758 // know what framerate to adapt form.
asapersson02465b82017-04-10 08:12:521759 const int kInputFps = 30;
sprangc5d62e22017-04-03 06:53:041760 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 08:12:521761 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-03 06:53:041762 stats_proxy_->SetMockStats(stats);
1763
mflodmancc3d4422017-08-03 15:27:511764 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:461765 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-03 06:53:041766 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:521767 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:041768 frame_timestamp += kFrameIntervalMs;
1769
1770 // Some framerate constraint should be set.
sprang84a37592017-02-10 15:04:271771 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-03 06:53:041772 EXPECT_EQ(std::numeric_limits<int>::max(),
1773 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:521774 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-03 06:53:041775
asapersson02465b82017-04-10 08:12:521776 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 21:20:411777 video_stream_encoder_->SetSource(&new_video_source,
1778 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 08:42:191779 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-03 06:53:041780
mflodmancc3d4422017-08-03 15:27:511781 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:041782 new_video_source.IncomingCapturedFrame(
1783 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:521784 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-03 06:53:041785 frame_timestamp += kFrameIntervalMs;
1786
1787 // Still no degradation.
Åsa Persson8c1bf952018-09-13 08:42:191788 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 18:45:461789
1790 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 15:27:511791 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411792 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-03 06:53:041793 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1794 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 15:04:271795 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 08:42:191796 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-03 06:53:041797
1798 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 15:27:511799 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411800 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:041801 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1802 EXPECT_EQ(std::numeric_limits<int>::max(),
1803 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 08:12:521804 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 18:45:461805
mflodmancc3d4422017-08-03 15:27:511806 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:461807}
1808
mflodmancc3d4422017-08-03 15:27:511809TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 13:14:011810 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551811 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1812 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 18:45:461813
asaperssonfab67072017-04-04 12:51:491814 const int kWidth = 1280;
1815 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:491816 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521817 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 12:51:491818 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1819 EXPECT_FALSE(stats.bw_limited_resolution);
1820 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1821
1822 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:511823 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:491824 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521825 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 12:51:491826
1827 stats = stats_proxy_->GetStats();
1828 EXPECT_TRUE(stats.bw_limited_resolution);
1829 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1830
1831 // Trigger adapt up.
mflodmancc3d4422017-08-03 15:27:511832 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:491833 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521834 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 12:51:491835
1836 stats = stats_proxy_->GetStats();
1837 EXPECT_FALSE(stats.bw_limited_resolution);
1838 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1839 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1840
mflodmancc3d4422017-08-03 15:27:511841 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 12:51:491842}
1843
mflodmancc3d4422017-08-03 15:27:511844TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 13:14:011845 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551846 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1847 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonfab67072017-04-04 12:51:491848
1849 const int kWidth = 1280;
1850 const int kHeight = 720;
asaperssonfab67072017-04-04 12:51:491851 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521852 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 18:45:461853 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1854 EXPECT_FALSE(stats.cpu_limited_resolution);
1855 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1856
1857 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:511858 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:491859 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521860 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 18:45:461861
1862 stats = stats_proxy_->GetStats();
1863 EXPECT_TRUE(stats.cpu_limited_resolution);
1864 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1865
1866 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 15:27:511867 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 12:51:491868 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521869 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 18:45:461870
1871 stats = stats_proxy_->GetStats();
1872 EXPECT_FALSE(stats.cpu_limited_resolution);
1873 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 12:51:491874 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:461875
mflodmancc3d4422017-08-03 15:27:511876 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:461877}
1878
mflodmancc3d4422017-08-03 15:27:511879TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 13:14:011880 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551881 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1882 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 09:44:111883
asaperssonfab67072017-04-04 12:51:491884 const int kWidth = 1280;
1885 const int kHeight = 720;
1886 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521887 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:111888 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:181889 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:111890 EXPECT_FALSE(stats.cpu_limited_resolution);
1891 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1892
asaperssonfab67072017-04-04 12:51:491893 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 15:27:511894 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:491895 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521896 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:111897 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:181898 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:111899 EXPECT_TRUE(stats.cpu_limited_resolution);
1900 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1901
1902 // Set new source with adaptation still enabled.
1903 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:511904 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411905 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:111906
asaperssonfab67072017-04-04 12:51:491907 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521908 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:111909 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:181910 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:111911 EXPECT_TRUE(stats.cpu_limited_resolution);
1912 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1913
1914 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 21:20:411915 video_stream_encoder_->SetSource(&new_video_source,
1916 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 09:44:111917
asaperssonfab67072017-04-04 12:51:491918 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521919 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:111920 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:181921 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:111922 EXPECT_FALSE(stats.cpu_limited_resolution);
1923 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1924
1925 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 15:27:511926 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411927 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 09:44:111928
asaperssonfab67072017-04-04 12:51:491929 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521930 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:111931 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:181932 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:111933 EXPECT_TRUE(stats.cpu_limited_resolution);
1934 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1935
asaperssonfab67072017-04-04 12:51:491936 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 15:27:511937 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 12:51:491938 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521939 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 09:44:111940 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:181941 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 09:44:111942 EXPECT_FALSE(stats.cpu_limited_resolution);
1943 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:521944 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:111945
mflodmancc3d4422017-08-03 15:27:511946 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:111947}
1948
mflodmancc3d4422017-08-03 15:27:511949TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 13:14:011950 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:551951 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1952 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 09:44:111953
asaperssonfab67072017-04-04 12:51:491954 const int kWidth = 1280;
1955 const int kHeight = 720;
1956 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521957 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:111958 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:111959 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:021960 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:491961 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:111962
1963 // Set new source with adaptation still enabled.
1964 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 21:20:411965 video_stream_encoder_->SetSource(&new_video_source,
1966 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:111967
asaperssonfab67072017-04-04 12:51:491968 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521969 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 09:44:111970 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:111971 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:021972 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:491973 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:111974
asaperssonfab67072017-04-04 12:51:491975 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:511976 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:491977 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521978 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:111979 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:111980 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:021981 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:491982 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:111983
asaperssonfab67072017-04-04 12:51:491984 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 21:20:411985 video_stream_encoder_->SetSource(&new_video_source,
1986 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 09:44:111987
asaperssonfab67072017-04-04 12:51:491988 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:521989 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:111990 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:111991 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:021992 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:491993 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 09:44:111994
asapersson02465b82017-04-10 08:12:521995 // Disable resolution scaling.
mflodmancc3d4422017-08-03 15:27:511996 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:411997 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:111998
asaperssonfab67072017-04-04 12:51:491999 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522000 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 09:44:112001 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 09:44:112002 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 07:01:022003 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 12:51:492004 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2005 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 09:44:112006
mflodmancc3d4422017-08-03 15:27:512007 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112008}
2009
mflodmancc3d4422017-08-03 15:27:512010TEST_F(VideoStreamEncoderTest,
2011 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 13:14:012012 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552013 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2014 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson36e9eb42017-03-31 12:29:122015
2016 const int kWidth = 1280;
2017 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 08:42:192018 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 12:29:122019 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 08:42:192020 video_source_.IncomingCapturedFrame(
2021 CreateFrame(timestamp_ms, kWidth, kHeight));
2022 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122023 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2025 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2026
2027 // Trigger adapt down.
mflodmancc3d4422017-08-03 15:27:512028 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:192029 timestamp_ms += kFrameIntervalMs;
2030 video_source_.IncomingCapturedFrame(
2031 CreateFrame(timestamp_ms, kWidth, kHeight));
2032 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122033 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2034 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2035 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2036
2037 // Trigger overuse.
mflodmancc3d4422017-08-03 15:27:512038 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:192039 timestamp_ms += kFrameIntervalMs;
2040 video_source_.IncomingCapturedFrame(
2041 CreateFrame(timestamp_ms, kWidth, kHeight));
2042 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122043 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2044 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2045 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2046
Niels Möller4db138e2018-04-19 07:04:132047 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 12:29:122048 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:132049
2050 VideoEncoderConfig video_encoder_config;
2051 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2052 // Make format different, to force recreation of encoder.
2053 video_encoder_config.video_format.parameters["foo"] = "foo";
2054 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:472055 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 08:42:192056 timestamp_ms += kFrameIntervalMs;
2057 video_source_.IncomingCapturedFrame(
2058 CreateFrame(timestamp_ms, kWidth, kHeight));
2059 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 12:29:122060 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2061 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2062 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2063
mflodmancc3d4422017-08-03 15:27:512064 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 12:29:122065}
2066
mflodmancc3d4422017-08-03 15:27:512067TEST_F(VideoStreamEncoderTest,
2068 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 13:14:012069 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552070 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2071 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 18:45:462072
asapersson0944a802017-04-07 07:57:582073 const int kWidth = 1280;
2074 const int kHeight = 720;
sprang84a37592017-02-10 15:04:272075 int sequence = 1;
perkj803d97f2016-11-01 18:45:462076
asaperssonfab67072017-04-04 12:51:492077 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522078 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462079 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 15:04:272080 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022081 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 15:04:272082 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2083
asapersson02465b82017-04-10 08:12:522084 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 15:27:512085 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 12:51:492086 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522087 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 15:04:272088 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 18:45:462089 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022090 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:462091 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2092
2093 // Set new source with adaptation still enabled.
2094 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512095 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412096 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 18:45:462097
2098 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492099 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522100 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462101 stats = stats_proxy_->GetStats();
2102 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:182103 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:462104 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2105
sprangc5d62e22017-04-03 06:53:042106 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 15:27:512107 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412108 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 18:45:462109 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492110 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522111 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462112 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-03 06:53:042113 // Not adapted at first.
perkj803d97f2016-11-01 18:45:462114 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-16 06:40:182115 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 18:45:462116 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2117
sprangc5d62e22017-04-03 06:53:042118 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-16 06:40:182119 // know what framerate to adapt from.
sprangc5d62e22017-04-03 06:53:042120 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2121 mock_stats.input_frame_rate = 30;
2122 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:512123 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042124 stats_proxy_->ResetMockStats();
2125
2126 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492127 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522128 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:042129
2130 // Framerate now adapted.
2131 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-16 06:40:182132 EXPECT_FALSE(stats.cpu_limited_resolution);
2133 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042134 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2135
2136 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 21:20:412137 video_stream_encoder_->SetSource(&new_video_source,
2138 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-03 06:53:042139 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492140 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522141 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:042142
2143 stats = stats_proxy_->GetStats();
2144 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022145 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042146 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2147
2148 // Try to trigger overuse. Should not succeed.
2149 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 15:27:512150 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:042151 stats_proxy_->ResetMockStats();
2152
2153 stats = stats_proxy_->GetStats();
2154 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022155 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042156 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2157
2158 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 15:27:512159 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412160 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 12:51:492161 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522162 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462163 stats = stats_proxy_->GetStats();
2164 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022165 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042166 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 18:45:462167
2168 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 15:27:512169 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 12:51:492170 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522171 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 18:45:462172 stats = stats_proxy_->GetStats();
2173 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022174 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042175 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2176
2177 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 15:27:512178 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412179 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:042180 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492181 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522182 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:042183 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 07:01:022184 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-03 06:53:042185 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022186 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042187 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2188
2189 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 15:27:512190 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-03 06:53:042191 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:492192 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522193 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-03 06:53:042194 stats = stats_proxy_->GetStats();
2195 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 07:01:022196 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-03 06:53:042197 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 08:12:522198 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 18:45:462199
mflodmancc3d4422017-08-03 15:27:512200 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 18:45:462201}
2202
mflodmancc3d4422017-08-03 15:27:512203TEST_F(VideoStreamEncoderTest,
2204 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 12:51:492205 const int kWidth = 1280;
2206 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012207 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552208 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2209 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 09:44:112210
asaperssonfab67072017-04-04 12:51:492211 // Expect no scaling to begin with.
asapersson02465b82017-04-10 08:12:522212 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 09:44:112213
asaperssonfab67072017-04-04 12:51:492214 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522215 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 09:44:112216
asaperssonfab67072017-04-04 12:51:492217 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:512218 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 11:59:512219
asaperssonfab67072017-04-04 12:51:492220 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522221 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 11:59:512222
kthelgason876222f2016-11-29 09:44:112223 // Expect a scale down.
2224 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 12:51:492225 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 09:44:112226
asapersson02465b82017-04-10 08:12:522227 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 09:44:112228 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512229 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412230 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 09:44:112231
asaperssonfab67072017-04-04 12:51:492232 // Trigger scale down.
mflodmancc3d4422017-08-03 15:27:512233 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 12:51:492234 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522235 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 09:44:112236
asaperssonfab67072017-04-04 12:51:492237 // Expect no scaling.
sprangc5d62e22017-04-03 06:53:042238 EXPECT_EQ(std::numeric_limits<int>::max(),
2239 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:112240
asaperssonfab67072017-04-04 12:51:492241 // Trigger scale up.
mflodmancc3d4422017-08-03 15:27:512242 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 12:51:492243 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522244 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 09:44:112245
asapersson02465b82017-04-10 08:12:522246 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-03 06:53:042247 EXPECT_EQ(std::numeric_limits<int>::max(),
2248 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 09:44:112249
mflodmancc3d4422017-08-03 15:27:512250 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 09:44:112251}
2252
mflodmancc3d4422017-08-03 15:27:512253TEST_F(VideoStreamEncoderTest,
2254 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:522255 const int kWidth = 1280;
2256 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012257 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552258 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2259 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 08:12:522260
Taylor Brandstetter49fcc102018-05-16 21:20:412261 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:522262 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:512263 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412264 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:522265
2266 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522267 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 08:42:192268 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522269 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2270 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2271
2272 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512273 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-16 06:40:182274 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 08:12:522275 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2276 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2277 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2278
2279 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 15:27:512280 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 08:12:522281 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2282 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2283 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2284
mflodmancc3d4422017-08-03 15:27:512285 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:522286}
2287
mflodmancc3d4422017-08-03 15:27:512288TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:222289 const int kWidth = 1280;
2290 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012291 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552292 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2293 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-14 06:25:222294
Taylor Brandstetter49fcc102018-05-16 21:20:412295 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:222296 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:412297 video_stream_encoder_->SetSource(&source,
2298 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:222299 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2300 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 08:42:192301 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222302
2303 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512304 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:222305 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2306 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2307 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2308 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2309
2310 // Trigger adapt down for same input resolution, expect no change.
2311 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2312 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:512313 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:222314 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2315 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2316 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2317
2318 // Trigger adapt down for larger input resolution, expect no change.
2319 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2320 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 15:27:512321 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:222322 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2323 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2324 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2325
mflodmancc3d4422017-08-03 15:27:512326 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222327}
2328
mflodmancc3d4422017-08-03 15:27:512329TEST_F(VideoStreamEncoderTest,
2330 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:522331 const int kWidth = 1280;
2332 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012333 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552334 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2335 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 08:12:522336
Taylor Brandstetter49fcc102018-05-16 21:20:412337 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:522338 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:512339 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412340 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:522341
2342 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522343 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192344 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522345 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2346 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2347
2348 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:512349 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:192350 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522351 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2352 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2353
mflodmancc3d4422017-08-03 15:27:512354 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:522355}
2356
mflodmancc3d4422017-08-03 15:27:512357TEST_F(VideoStreamEncoderTest,
2358 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 08:12:522359 const int kWidth = 1280;
2360 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012361 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552362 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2363 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 08:12:522364
Taylor Brandstetter49fcc102018-05-16 21:20:412365 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 08:12:522366 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:512367 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412368 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 08:12:522369
2370 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522371 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192372 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-16 06:40:182373 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:522374 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2375
2376 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:512377 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:192378 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-16 06:40:182379 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 08:12:522380 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2381
mflodmancc3d4422017-08-03 15:27:512382 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:522383}
2384
mflodmancc3d4422017-08-03 15:27:512385TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:222386 const int kWidth = 1280;
2387 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012388 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552389 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2390 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-14 06:25:222391
Taylor Brandstetter49fcc102018-05-16 21:20:412392 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:222393 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:412394 video_stream_encoder_->SetSource(&source,
2395 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:222396
2397 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2398 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192399 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222400 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2401 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2402 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2403
2404 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:512405 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192406 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222407 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2408 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2409 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2410
mflodmancc3d4422017-08-03 15:27:512411 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222412}
2413
mflodmancc3d4422017-08-03 15:27:512414TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-16 06:40:182415 const int kWidth = 1280;
2416 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012417 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552418 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2419 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-16 06:40:182420
Taylor Brandstetter49fcc102018-05-16 21:20:412421 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-16 06:40:182422 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:412423 video_stream_encoder_->SetSource(&source,
2424 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-16 06:40:182425
2426 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2427 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192428 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-16 06:40:182429 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2430 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2431 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2432
2433 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:512434 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192435 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-16 06:40:182436 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2437 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2438 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2439
mflodmancc3d4422017-08-03 15:27:512440 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:182441}
2442
mflodmancc3d4422017-08-03 15:27:512443TEST_F(VideoStreamEncoderTest,
2444 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 08:12:522445 const int kWidth = 1280;
2446 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012447 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552448 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2449 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 08:12:522450
Taylor Brandstetter49fcc102018-05-16 21:20:412451 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 08:12:522452 AdaptingFrameForwarder source;
2453 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:512454 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412455 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 08:12:522456
2457 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522458 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 08:42:192459 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522460 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2461 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2462
2463 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512464 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 08:12:522465 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522466 WaitForEncodedFrame(2);
asapersson09f05612017-05-16 06:40:182467 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 08:12:522468 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2469 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2470
2471 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512472 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192473 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 08:12:522474 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2475 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2476 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2477
mflodmancc3d4422017-08-03 15:27:512478 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:522479}
2480
mflodmancc3d4422017-08-03 15:27:512481TEST_F(VideoStreamEncoderTest,
2482 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-16 06:40:182483 const int kWidth = 1280;
2484 const int kHeight = 720;
2485 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 13:14:012486 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552487 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2488 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-16 06:40:182489
2490 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2491 stats.input_frame_rate = kInputFps;
2492 stats_proxy_->SetMockStats(stats);
2493
Taylor Brandstetter49fcc102018-05-16 21:20:412494 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-16 06:40:182495 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2496 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 08:42:192497 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-16 06:40:182498
2499 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512500 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-16 06:40:182501 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2502 sink_.WaitForEncodedFrame(2);
2503 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2504
Taylor Brandstetter49fcc102018-05-16 21:20:412505 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-16 06:40:182506 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 15:27:512507 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412508 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 08:42:192509 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-16 06:40:182510
2511 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 15:27:512512 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-16 06:40:182513 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2514 sink_.WaitForEncodedFrame(3);
2515 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2516
2517 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512518 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192519 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-16 06:40:182520
mflodmancc3d4422017-08-03 15:27:512521 video_stream_encoder_->Stop();
asapersson09f05612017-05-16 06:40:182522}
2523
mflodmancc3d4422017-08-03 15:27:512524TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 08:47:312525 const int kWidth = 1280;
2526 const int kHeight = 720;
2527 const size_t kNumFrames = 10;
2528
Erik Språng4c6ca302019-04-08 13:14:012529 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552530 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2531 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason5e13d412016-12-01 11:59:512532
asaperssond0de2952017-04-21 08:47:312533 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 15:58:542534 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 08:47:312535 video_source_.set_adaptation_enabled(true);
2536
2537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2538 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2539
2540 int downscales = 0;
2541 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 08:42:192542 video_source_.IncomingCapturedFrame(
2543 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2544 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 08:47:312545
asaperssonfab67072017-04-04 12:51:492546 // Trigger scale down.
asaperssond0de2952017-04-21 08:47:312547 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 15:27:512548 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-03 06:53:042549 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 08:47:312550
2551 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2552 ++downscales;
2553
2554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2555 EXPECT_EQ(downscales,
2556 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2557 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 11:59:512558 }
mflodmancc3d4422017-08-03 15:27:512559 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:312560}
2561
mflodmancc3d4422017-08-03 15:27:512562TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:312563 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2564 const int kWidth = 1280;
2565 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012566 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552567 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2568 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 08:47:312569
Taylor Brandstetter49fcc102018-05-16 21:20:412570 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:312571 AdaptingFrameForwarder source;
2572 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:512573 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:412574 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:312575
Åsa Persson8c1bf952018-09-13 08:42:192576 int64_t timestamp_ms = kFrameIntervalMs;
2577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522578 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192579 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:312580 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2581 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2582
2583 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512584 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:192585 timestamp_ms += kFrameIntervalMs;
2586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2587 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:182588 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 08:47:312589 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2590 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2591
2592 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512593 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:192594 timestamp_ms += kFrameIntervalMs;
2595 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:522596 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192597 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:312598 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2599 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2600
2601 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512602 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:192603 timestamp_ms += kFrameIntervalMs;
2604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2605 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:182606 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 08:47:312607 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2608 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2609
2610 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512611 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:192612 timestamp_ms += kFrameIntervalMs;
2613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-16 06:40:182614 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192615 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:312616 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2617 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2618
mflodmancc3d4422017-08-03 15:27:512619 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:312620}
2621
mflodmancc3d4422017-08-03 15:27:512622TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-14 06:25:222623 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2624 const int kWidth = 1280;
2625 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:012626 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:552627 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2628 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-14 06:25:222629
Taylor Brandstetter49fcc102018-05-16 21:20:412630 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:222631 AdaptingFrameForwarder source;
2632 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:412633 video_stream_encoder_->SetSource(&source,
2634 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:222635
Åsa Persson8c1bf952018-09-13 08:42:192636 int64_t timestamp_ms = kFrameIntervalMs;
2637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:222638 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192639 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222640 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2641 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2642
2643 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512644 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:192645 timestamp_ms += kFrameIntervalMs;
2646 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2647 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:222648 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2649 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2650 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2651
2652 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512653 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192654 timestamp_ms += kFrameIntervalMs;
2655 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:222656 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192657 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222658 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2659 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2660
2661 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 15:27:512662 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:192663 timestamp_ms += kFrameIntervalMs;
2664 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2665 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:222666 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2667 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2668 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2669
2670 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 15:27:512671 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:192672 timestamp_ms += kFrameIntervalMs;
2673 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-14 06:25:222674 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:192675 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:222676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2677 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2678
mflodmancc3d4422017-08-03 15:27:512679 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:222680}
2681
Sergey Silkin41c650b2019-10-14 11:12:192682TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
2683 fake_encoder_.SetResolutionBitrateLimits(
2684 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2685
2686 video_stream_encoder_->OnBitrateUpdated(
2687 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2688 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2689 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, 0);
2690
2691 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2692 AdaptingFrameForwarder source;
2693 source.set_adaptation_enabled(true);
2694 video_stream_encoder_->SetSource(
2695 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2696
2697 // Insert 720p frame.
2698 int64_t timestamp_ms = kFrameIntervalMs;
2699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2700 WaitForEncodedFrame(1280, 720);
2701
2702 // Reduce bitrate and trigger adapt down.
2703 video_stream_encoder_->OnBitrateUpdated(
2704 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2705 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2706 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, 0);
2707 video_stream_encoder_->TriggerQualityLow();
2708
2709 // Insert 720p frame. It should be downscaled and encoded.
2710 timestamp_ms += kFrameIntervalMs;
2711 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2712 WaitForEncodedFrame(960, 540);
2713
2714 // Trigger adapt up. Higher resolution should not be requested duo to lack
2715 // of bitrate.
2716 video_stream_encoder_->TriggerQualityHigh();
2717 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2718
2719 // Increase bitrate.
2720 video_stream_encoder_->OnBitrateUpdated(
2721 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2722 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2723 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, 0);
2724
2725 // Trigger adapt up. Higher resolution should be requested.
2726 video_stream_encoder_->TriggerQualityHigh();
2727 VerifyFpsMaxResolutionMax(source.sink_wants());
2728
2729 video_stream_encoder_->Stop();
2730}
2731
2732TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
2733 fake_encoder_.SetResolutionBitrateLimits(
2734 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2735
2736 // Set bitrate equal to min bitrate of 540p.
2737 video_stream_encoder_->OnBitrateUpdated(
2738 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2739 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2740 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, 0);
2741
2742 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2743 AdaptingFrameForwarder source;
2744 source.set_adaptation_enabled(true);
2745 video_stream_encoder_->SetSource(
2746 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2747
2748 // Insert 720p frame. It should be dropped and lower resolution should be
2749 // requested.
2750 int64_t timestamp_ms = kFrameIntervalMs;
2751 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2752 ExpectDroppedFrame();
2753 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2754
2755 // Insert 720p frame. It should be downscaled and encoded.
2756 timestamp_ms += kFrameIntervalMs;
2757 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2758 WaitForEncodedFrame(960, 540);
2759
2760 video_stream_encoder_->Stop();
2761}
2762
Åsa Perssonb67c44c2019-09-24 13:25:322763class BalancedDegradationTest : public VideoStreamEncoderTest {
2764 protected:
2765 void SetupTest() {
2766 // Reset encoder for field trials to take effect.
2767 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 13:13:042768 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 13:25:322769
2770 // Enable BALANCED preference.
2771 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 13:13:042772 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
2773 }
2774
2775 void OnBitrateUpdated(int bitrate_bps) {
2776 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(bitrate_bps),
2777 DataRate::bps(bitrate_bps),
2778 DataRate::bps(bitrate_bps), 0, 0);
Åsa Perssonb67c44c2019-09-24 13:25:322779 }
2780
Åsa Persson45b176f2019-09-30 09:19:052781 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 13:25:322782 timestamp_ms_ += kFrameIntervalMs;
2783 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 09:19:052784 }
2785
2786 void InsertFrameAndWaitForEncoded() {
2787 InsertFrame();
Åsa Perssonb67c44c2019-09-24 13:25:322788 sink_.WaitForEncodedFrame(timestamp_ms_);
2789 }
2790
2791 const int kWidth = 640; // pixels:640x360=230400
2792 const int kHeight = 360;
2793 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
2794 int64_t timestamp_ms_ = 0;
2795 AdaptingFrameForwarder source_;
2796};
2797
2798TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
2799 test::ScopedFieldTrials field_trials(
2800 "WebRTC-Video-BalancedDegradationSettings/"
2801 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
2802 SetupTest();
2803
2804 // Force input frame rate.
2805 const int kInputFps = 24;
2806 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2807 stats.input_frame_rate = kInputFps;
2808 stats_proxy_->SetMockStats(stats);
2809
Åsa Persson45b176f2019-09-30 09:19:052810 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 13:25:322811 VerifyFpsMaxResolutionMax(source_.sink_wants());
2812
2813 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
2814 // Fps diff (input-requested:0) < threshold, expect AdaptDown to return false.
2815 video_stream_encoder_->TriggerQualityLowExpectFalse();
2816 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
2817
2818 video_stream_encoder_->Stop();
2819}
2820
2821TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
2822 test::ScopedFieldTrials field_trials(
2823 "WebRTC-Video-BalancedDegradationSettings/"
2824 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
2825 SetupTest();
2826
2827 // Force input frame rate.
2828 const int kInputFps = 25;
2829 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2830 stats.input_frame_rate = kInputFps;
2831 stats_proxy_->SetMockStats(stats);
2832
Åsa Persson45b176f2019-09-30 09:19:052833 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 13:25:322834 VerifyFpsMaxResolutionMax(source_.sink_wants());
2835
2836 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
2837 // Fps diff (input-requested:1) == threshold, expect AdaptDown to return true.
2838 video_stream_encoder_->TriggerQualityLow();
2839 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
2840
2841 video_stream_encoder_->Stop();
2842}
2843
2844TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
2845 test::ScopedFieldTrials field_trials(
2846 "WebRTC-Video-BalancedDegradationSettings/"
2847 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
2848 SetupTest();
2849
2850 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
2851
Åsa Persson45b176f2019-09-30 09:19:052852 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 13:25:322853 VerifyFpsMaxResolutionMax(source_.sink_wants());
2854
2855 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
2856 video_stream_encoder_->TriggerQualityLow();
2857 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
2858
2859 video_stream_encoder_->Stop();
2860}
2861
Åsa Perssonccfb3402019-09-25 13:13:042862TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:322863 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 15:26:392864 "WebRTC-Video-BalancedDegradationSettings/"
2865 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 13:13:042866 SetupTest();
Åsa Persson1b247f12019-08-14 15:26:392867
Åsa Persson1b247f12019-08-14 15:26:392868 const int kMinBitrateBps = 425000;
2869 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 13:13:042870 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 15:26:392871
Åsa Persson45b176f2019-09-30 09:19:052872 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042873 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 15:26:392874 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2875
2876 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2877 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:052878 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042879 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 15:26:392880 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2881
2882 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2883 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:052884 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042885 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 15:26:392886 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2887
Åsa Persson30ab0152019-08-27 10:22:332888 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2889 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:052890 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042891 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
2892 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 10:22:332893 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2894
2895 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 15:26:392896 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:052897 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:332898 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 15:26:392899
Åsa Persson30ab0152019-08-27 10:22:332900 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:042901 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 15:26:392902 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:052903 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042904 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 10:22:332905 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2906
2907 video_stream_encoder_->Stop();
2908}
2909
Åsa Perssonccfb3402019-09-25 13:13:042910TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 09:19:052911 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
2912 test::ScopedFieldTrials field_trials(
2913 "WebRTC-Video-BalancedDegradationSettings/"
2914 "pixels:57600|129600|230400,fps:7|24|24/");
2915 SetupTest();
2916 OnBitrateUpdated(kLowTargetBitrateBps);
2917
2918 VerifyNoLimitation(source_.sink_wants());
2919
2920 // Insert frame, expect scaled down:
2921 // framerate (640x360@24fps) -> resolution (480x270@24fps).
2922 InsertFrame();
2923 EXPECT_FALSE(WaitForFrame(1000));
2924 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
2925 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
2926
2927 // Insert frame, expect scaled down:
2928 // resolution (320x180@24fps).
2929 InsertFrame();
2930 EXPECT_FALSE(WaitForFrame(1000));
2931 EXPECT_LT(source_.sink_wants().max_pixel_count,
2932 source_.last_wants().max_pixel_count);
2933 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
2934
2935 // Frame should not be dropped (min pixels per frame reached).
2936 InsertFrameAndWaitForEncoded();
2937
2938 video_stream_encoder_->Stop();
2939}
2940
2941TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:332942 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:322943 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:332944 "WebRTC-Video-BalancedDegradationSettings/"
2945 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:042946 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:332947
Åsa Persson30ab0152019-08-27 10:22:332948 const int kResolutionMinBitrateBps = 435000;
2949 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 13:13:042950 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:332951
Åsa Persson45b176f2019-09-30 09:19:052952 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042953 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 10:22:332954 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2955
2956 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2957 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:052958 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042959 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 10:22:332960 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2961
2962 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2963 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:052964 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042965 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:332966 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2967
2968 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2969 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:052970 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042971 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 15:26:392972 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2973
Åsa Persson30ab0152019-08-27 10:22:332974 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
2975 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:052976 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042977 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:332978 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2979
2980 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
2981 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:052982 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:332983 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2984
2985 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:042986 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:332987 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:052988 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:042989 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:332990 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2991
2992 video_stream_encoder_->Stop();
2993}
2994
Åsa Perssonccfb3402019-09-25 13:13:042995TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 10:22:332996 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 13:25:322997 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 10:22:332998 "WebRTC-Video-BalancedDegradationSettings/"
2999 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 13:13:043000 SetupTest();
Åsa Persson30ab0152019-08-27 10:22:333001
Åsa Persson30ab0152019-08-27 10:22:333002 const int kMinBitrateBps = 425000;
3003 const int kTooLowMinBitrateBps = 424000;
3004 const int kResolutionMinBitrateBps = 435000;
3005 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 13:13:043006 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333007
Åsa Persson45b176f2019-09-30 09:19:053008 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043009 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 10:22:333010 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3011
3012 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3013 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053014 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043015 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 10:22:333016 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3017
3018 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3019 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053020 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043021 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333022 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3023
3024 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3025 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 09:19:053026 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043027 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333028 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3029
3030 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3031 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053032 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:333033 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3034
3035 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043036 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333037 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053038 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043039 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333040 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3041
3042 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043043 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333044 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053045 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 10:22:333046 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3047
3048 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 13:13:043049 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 10:22:333050 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 09:19:053051 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 13:13:043052 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 10:22:333053 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3054
Åsa Persson1b247f12019-08-14 15:26:393055 video_stream_encoder_->Stop();
3056}
3057
mflodmancc3d4422017-08-03 15:27:513058TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:313059 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3060 const int kWidth = 1280;
3061 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 13:14:013062 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553063 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3064 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 08:47:313065
Taylor Brandstetter49fcc102018-05-16 21:20:413066 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:313067 AdaptingFrameForwarder source;
3068 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 15:27:513069 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413070 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:313071
Åsa Persson8c1bf952018-09-13 08:42:193072 int64_t timestamp_ms = kFrameIntervalMs;
3073 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523074 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:193075 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:313076 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3077 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3078 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3079 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3080
3081 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 15:27:513082 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193083 timestamp_ms += kFrameIntervalMs;
3084 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3085 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183086 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 08:47:313087 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3088 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3089 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3090 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3091
3092 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 15:27:513093 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193094 timestamp_ms += kFrameIntervalMs;
3095 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3096 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183097 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 08:47:313098 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3099 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3100 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3101 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3102
Jonathan Yubc771b72017-12-09 01:04:293103 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 15:27:513104 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193105 timestamp_ms += kFrameIntervalMs;
3106 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3107 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-09 01:04:293108 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 08:47:313109 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3110 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:293111 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:313112 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3113
Jonathan Yubc771b72017-12-09 01:04:293114 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 15:27:513115 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193116 timestamp_ms += kFrameIntervalMs;
3117 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3118 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183119 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-09 01:04:293120 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 08:47:313121 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3122 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3123 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3124 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3125
Jonathan Yubc771b72017-12-09 01:04:293126 // Trigger quality adapt down, expect no change (min resolution reached).
3127 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 08:42:193128 timestamp_ms += kFrameIntervalMs;
3129 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3130 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-09 01:04:293131 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3132 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3133 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3134 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3135 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3136
3137 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 15:27:513138 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:193139 timestamp_ms += kFrameIntervalMs;
3140 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3141 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183142 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-09 01:04:293143 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3144 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3145 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3146 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3147
3148 // Trigger cpu adapt up, expect upscaled resolution (640x360).
3149 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:193150 timestamp_ms += kFrameIntervalMs;
3151 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3152 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-09 01:04:293153 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3154 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3155 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3156 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3157 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3158
3159 // Trigger cpu adapt up, expect upscaled resolution (960x540).
3160 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:193161 timestamp_ms += kFrameIntervalMs;
3162 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3163 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-09 01:04:293164 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 08:47:313165 last_wants = source.sink_wants();
3166 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3167 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:293168 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:313169 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3170
3171 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 15:27:513172 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 08:42:193173 timestamp_ms += kFrameIntervalMs;
3174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3175 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-16 06:40:183176 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 08:47:313177 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3178 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:293179 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:313180 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3181
3182 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 15:27:513183 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:193184 timestamp_ms += kFrameIntervalMs;
3185 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523186 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-16 06:40:183187 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 08:42:193188 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:313189 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3190 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-09 01:04:293191 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 08:47:313192 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 11:59:513193
mflodmancc3d4422017-08-03 15:27:513194 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 11:59:513195}
3196
mflodmancc3d4422017-08-03 15:27:513197TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 12:51:493198 const int kWidth = 640;
3199 const int kHeight = 360;
perkj803d97f2016-11-01 18:45:463200
Erik Språng4c6ca302019-04-08 13:14:013201 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553202 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3203 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 14:06:523204
perkj803d97f2016-11-01 18:45:463205 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:493206 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523207 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 18:45:463208 }
3209
mflodmancc3d4422017-08-03 15:27:513210 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 18:45:463211 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 12:51:493212 video_source_.IncomingCapturedFrame(CreateFrame(
3213 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523214 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 18:45:463215 }
3216
mflodmancc3d4422017-08-03 15:27:513217 video_stream_encoder_->Stop();
3218 video_stream_encoder_.reset();
perkj803d97f2016-11-01 18:45:463219 stats_proxy_.reset();
sprangf8ee65e2017-02-28 16:49:333220
Ying Wangef3998f2019-12-09 12:06:533221 EXPECT_METRIC_EQ(
3222 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3223 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 18:45:463224 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3225}
3226
mflodmancc3d4422017-08-03 15:27:513227TEST_F(VideoStreamEncoderTest,
3228 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 13:14:013229 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553230 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3231 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf4e44af2017-04-19 09:01:063232 const int kWidth = 640;
3233 const int kHeight = 360;
3234
Taylor Brandstetter49fcc102018-05-16 21:20:413235 video_stream_encoder_->SetSource(&video_source_,
3236 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 09:01:063237
3238 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3239 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523240 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 09:01:063241 }
3242
mflodmancc3d4422017-08-03 15:27:513243 video_stream_encoder_->Stop();
3244 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 09:01:063245 stats_proxy_.reset();
3246
3247 EXPECT_EQ(0,
3248 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3249}
3250
mflodmancc3d4422017-08-03 15:27:513251TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 14:06:523252 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 12:09:313253 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 14:24:023254
3255 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 10:32:223256 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 14:24:023257 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 13:16:283258 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3259 kDefaultFps));
sprang57c2fff2017-01-16 14:24:023260
sprang57c2fff2017-01-16 14:24:023261 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 10:11:103262 .Times(1);
Florent Castellia8336d32019-09-09 11:36:553263 video_stream_encoder_->OnBitrateUpdated(
3264 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3265 DataRate::bps(kLowTargetBitrateBps), 0, 0);
sprang57c2fff2017-01-16 14:24:023266
sprang57c2fff2017-01-16 14:24:023267 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 20:19:533268 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3269 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 13:53:113270 VideoBitrateAllocation bitrate_allocation =
3271 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 20:19:533272 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 13:53:113273 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 10:11:203274 // TODO(srte): The use of millisecs here looks like an error, but the tests
3275 // fails using seconds, this should be investigated.
3276 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:023277
3278 // Not called on second frame.
3279 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3280 .Times(0);
3281 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 20:19:533282 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3283 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 10:11:203284 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 14:24:023285
3286 // Called after a process interval.
sprang57c2fff2017-01-16 14:24:023287 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3288 .Times(1);
Erik Språngd7329ca2019-02-21 20:19:533289 const int64_t start_time_ms = rtc::TimeMillis();
3290 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3291 video_source_.IncomingCapturedFrame(
3292 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3293 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 10:11:203294 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 20:19:533295 }
3296
3297 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 13:53:113298 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 14:24:023299
mflodmancc3d4422017-08-03 15:27:513300 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 14:24:023301}
3302
Åsa Perssonc29cb2c2019-03-25 11:06:593303TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3304 // 2 TLs configured, temporal layers supported by encoder.
3305 const int kNumTemporalLayers = 2;
3306 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3307 fake_encoder_.SetTemporalLayersSupported(0, true);
3308
3309 // Bitrate allocated across temporal layers.
3310 const int kTl0Bps = kTargetBitrateBps *
3311 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:463312 kNumTemporalLayers, /*temporal_id*/ 0,
3313 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:593314 const int kTl1Bps = kTargetBitrateBps *
3315 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 12:01:463316 kNumTemporalLayers, /*temporal_id*/ 1,
3317 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:593318 VideoBitrateAllocation expected_bitrate;
3319 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3320 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3321
3322 VerifyAllocatedBitrate(expected_bitrate);
3323 video_stream_encoder_->Stop();
3324}
3325
3326TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3327 // 2 TLs configured, temporal layers not supported by encoder.
3328 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3329 fake_encoder_.SetTemporalLayersSupported(0, false);
3330
3331 // Temporal layers not supported by the encoder.
3332 // Total bitrate should be at ti:0.
3333 VideoBitrateAllocation expected_bitrate;
3334 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3335
3336 VerifyAllocatedBitrate(expected_bitrate);
3337 video_stream_encoder_->Stop();
3338}
3339
3340TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3341 // 2 TLs configured, temporal layers only supported for first stream.
3342 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3343 fake_encoder_.SetTemporalLayersSupported(0, true);
3344 fake_encoder_.SetTemporalLayersSupported(1, false);
3345
3346 const int kS0Bps = 150000;
3347 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:463348 kS0Bps *
3349 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3350 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:593351 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 12:01:463352 kS0Bps *
3353 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3354 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 11:06:593355 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3356 // Temporal layers not supported by si:1.
3357 VideoBitrateAllocation expected_bitrate;
3358 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3359 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3360 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3361
3362 VerifyAllocatedBitrate(expected_bitrate);
3363 video_stream_encoder_->Stop();
3364}
3365
Niels Möller7dc26b72017-12-06 09:27:483366TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3367 const int kFrameWidth = 1280;
3368 const int kFrameHeight = 720;
3369 const int kFramerate = 24;
3370
Erik Språng4c6ca302019-04-08 13:14:013371 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553372 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3373 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 09:27:483374 test::FrameForwarder source;
3375 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413376 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:483377
3378 // Insert a single frame, triggering initial configuration.
3379 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3380 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3381
3382 EXPECT_EQ(
3383 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3384 kDefaultFramerate);
3385
3386 // Trigger reconfigure encoder (without resetting the entire instance).
3387 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:513388 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 09:27:483389 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3390 video_encoder_config.number_of_streams = 1;
3391 video_encoder_config.video_stream_factory =
3392 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3393 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473394 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:483395 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3396
3397 // Detector should be updated with fps limit from codec config.
3398 EXPECT_EQ(
3399 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3400 kFramerate);
3401
3402 // Trigger overuse, max framerate should be reduced.
3403 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3404 stats.input_frame_rate = kFramerate;
3405 stats_proxy_->SetMockStats(stats);
3406 video_stream_encoder_->TriggerCpuOveruse();
3407 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3408 int adapted_framerate =
3409 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3410 EXPECT_LT(adapted_framerate, kFramerate);
3411
3412 // Trigger underuse, max framerate should go back to codec configured fps.
3413 // Set extra low fps, to make sure it's actually reset, not just incremented.
3414 stats = stats_proxy_->GetStats();
3415 stats.input_frame_rate = adapted_framerate / 2;
3416 stats_proxy_->SetMockStats(stats);
3417 video_stream_encoder_->TriggerCpuNormalUsage();
3418 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3419 EXPECT_EQ(
3420 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3421 kFramerate);
3422
3423 video_stream_encoder_->Stop();
3424}
3425
3426TEST_F(VideoStreamEncoderTest,
3427 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3428 const int kFrameWidth = 1280;
3429 const int kFrameHeight = 720;
3430 const int kLowFramerate = 15;
3431 const int kHighFramerate = 25;
3432
Erik Språng4c6ca302019-04-08 13:14:013433 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553434 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3435 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 09:27:483436 test::FrameForwarder source;
3437 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413438 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 09:27:483439
3440 // Trigger initial configuration.
3441 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:513442 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 09:27:483443 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3444 video_encoder_config.number_of_streams = 1;
3445 video_encoder_config.video_stream_factory =
3446 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3447 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3448 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473449 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:483450 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3451
3452 EXPECT_EQ(
3453 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3454 kLowFramerate);
3455
3456 // Trigger overuse, max framerate should be reduced.
3457 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3458 stats.input_frame_rate = kLowFramerate;
3459 stats_proxy_->SetMockStats(stats);
3460 video_stream_encoder_->TriggerCpuOveruse();
3461 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3462 int adapted_framerate =
3463 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3464 EXPECT_LT(adapted_framerate, kLowFramerate);
3465
3466 // Reconfigure the encoder with a new (higher max framerate), max fps should
3467 // still respect the adaptation.
3468 video_encoder_config.video_stream_factory =
3469 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3470 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3471 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473472 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 09:27:483473 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3474
3475 EXPECT_EQ(
3476 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3477 adapted_framerate);
3478
3479 // Trigger underuse, max framerate should go back to codec configured fps.
3480 stats = stats_proxy_->GetStats();
3481 stats.input_frame_rate = adapted_framerate;
3482 stats_proxy_->SetMockStats(stats);
3483 video_stream_encoder_->TriggerCpuNormalUsage();
3484 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3485 EXPECT_EQ(
3486 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3487 kHighFramerate);
3488
3489 video_stream_encoder_->Stop();
3490}
3491
mflodmancc3d4422017-08-03 15:27:513492TEST_F(VideoStreamEncoderTest,
3493 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 11:21:073494 const int kFrameWidth = 1280;
3495 const int kFrameHeight = 720;
3496 const int kFramerate = 24;
3497
Erik Språng4c6ca302019-04-08 13:14:013498 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553499 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3500 DataRate::bps(kTargetBitrateBps), 0, 0);
sprangfda496a2017-06-15 11:21:073501 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513502 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413503 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 11:21:073504
3505 // Trigger initial configuration.
3506 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:513507 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 11:21:073508 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3509 video_encoder_config.number_of_streams = 1;
3510 video_encoder_config.video_stream_factory =
3511 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3512 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 15:27:513513 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473514 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:513515 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 11:21:073516
Niels Möller7dc26b72017-12-06 09:27:483517 EXPECT_EQ(
3518 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3519 kFramerate);
sprangfda496a2017-06-15 11:21:073520
3521 // Trigger overuse, max framerate should be reduced.
3522 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3523 stats.input_frame_rate = kFramerate;
3524 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 15:27:513525 video_stream_encoder_->TriggerCpuOveruse();
3526 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:483527 int adapted_framerate =
3528 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 11:21:073529 EXPECT_LT(adapted_framerate, kFramerate);
3530
3531 // Change degradation preference to not enable framerate scaling. Target
3532 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 15:27:513533 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413534 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 15:27:513535 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 09:27:483536 EXPECT_EQ(
3537 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3538 kFramerate);
sprangfda496a2017-06-15 11:21:073539
mflodmancc3d4422017-08-03 15:27:513540 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 11:21:073541}
3542
mflodmancc3d4422017-08-03 15:27:513543TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:493544 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 14:37:333545 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 13:14:013546 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 11:36:553547 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 13:14:013548 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 12:51:493549 const int kWidth = 640;
3550 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:223551
asaperssonfab67072017-04-04 12:51:493552 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:223553
3554 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:523555 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:223556
3557 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 07:57:583558 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:223559
sprangc5d62e22017-04-03 06:53:043560 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 15:02:223561
asaperssonfab67072017-04-04 12:51:493562 // Next frame is scaled.
kthelgason2bc68642017-02-07 15:02:223563 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 12:51:493564 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 15:02:223565
3566 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 14:06:523567 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:223568
sprangc5d62e22017-04-03 06:53:043569 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 15:02:223570
mflodmancc3d4422017-08-03 15:27:513571 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:223572}
3573
mflodmancc3d4422017-08-03 15:27:513574TEST_F(VideoStreamEncoderTest,
3575 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 12:51:493576 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 14:37:333577 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 13:14:013578 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 11:36:553579 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 13:14:013580 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 12:51:493581 const int kWidth = 640;
3582 const int kHeight = 360;
kthelgason2bc68642017-02-07 15:02:223583
3584 // We expect the n initial frames to get dropped.
3585 int i;
3586 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 12:51:493587 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523588 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 15:02:223589 }
3590 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 12:51:493591 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:523592 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 15:02:223593
3594 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 12:51:493595 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 15:02:223596
mflodmancc3d4422017-08-03 15:27:513597 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:223598}
3599
mflodmancc3d4422017-08-03 15:27:513600TEST_F(VideoStreamEncoderTest,
3601 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 12:51:493602 const int kWidth = 640;
3603 const int kHeight = 360;
Florent Castellia8336d32019-09-09 11:36:553604 video_stream_encoder_->OnBitrateUpdated(
3605 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3606 DataRate::bps(kLowTargetBitrateBps), 0, 0);
kthelgason2bc68642017-02-07 15:02:223607
3608 // Set degradation preference.
mflodmancc3d4422017-08-03 15:27:513609 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413610 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 15:02:223611
asaperssonfab67072017-04-04 12:51:493612 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 15:02:223613 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:523614 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 15:02:223615
mflodmancc3d4422017-08-03 15:27:513616 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 15:02:223617}
3618
mflodmancc3d4422017-08-03 15:27:513619TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 12:51:493620 const int kWidth = 640;
3621 const int kHeight = 360;
kthelgasonad9010c2017-02-14 08:46:513622 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 07:04:133623
3624 VideoEncoderConfig video_encoder_config;
3625 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3626 // Make format different, to force recreation of encoder.
3627 video_encoder_config.video_format.parameters["foo"] = "foo";
3628 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:473629 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 11:36:553630 video_stream_encoder_->OnBitrateUpdated(
3631 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3632 DataRate::bps(kLowTargetBitrateBps), 0, 0);
asapersson09f05612017-05-16 06:40:183633
kthelgasonb83797b2017-02-14 19:57:253634 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 21:20:413635 video_stream_encoder_->SetSource(&video_source_,
3636 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 08:46:513637
asaperssonfab67072017-04-04 12:51:493638 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 08:46:513639 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 14:06:523640 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 08:46:513641
mflodmancc3d4422017-08-03 15:27:513642 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 08:46:513643 fake_encoder_.SetQualityScaling(true);
3644}
3645
Kári Tristan Helgason639602a2018-08-02 08:51:403646TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
3647 webrtc::test::ScopedFieldTrials field_trials(
3648 "WebRTC-InitialFramedrop/Enabled/");
3649 // Reset encoder for field trials to take effect.
3650 ConfigureEncoder(video_encoder_config_.Copy());
3651 const int kTooLowBitrateForFrameSizeBps = 10000;
3652 const int kWidth = 640;
3653 const int kHeight = 360;
3654
Erik Språng4c6ca302019-04-08 13:14:013655 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553656 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3657 DataRate::bps(kTargetBitrateBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 08:51:403658 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3659 // Frame should not be dropped.
3660 WaitForEncodedFrame(1);
3661
Erik Språng610c7632019-03-06 14:37:333662 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 13:14:013663 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 11:36:553664 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 13:14:013665 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 08:51:403666 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3667 // Expect to drop this frame, the wait should time out.
3668 ExpectDroppedFrame();
3669
3670 // Expect the sink_wants to specify a scaled frame.
3671 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3672 video_stream_encoder_->Stop();
3673}
3674
Åsa Persson139f4dc2019-08-02 07:29:583675TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3676 webrtc::test::ScopedFieldTrials field_trials(
3677 "WebRTC-Video-QualityScalerSettings/"
3678 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3679 // Reset encoder for field trials to take effect.
3680 ConfigureEncoder(video_encoder_config_.Copy());
3681 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3682 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3683 const int kWidth = 640;
3684 const int kHeight = 360;
3685
3686 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553687 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3688 DataRate::bps(kTargetBitrateBps), 0, 0);
Åsa Persson139f4dc2019-08-02 07:29:583689 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3690 // Frame should not be dropped.
3691 WaitForEncodedFrame(1);
3692
3693 video_stream_encoder_->OnBitrateUpdated(
3694 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 11:36:553695 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
Åsa Persson139f4dc2019-08-02 07:29:583696 DataRate::bps(kNotTooLowBitrateForFrameSizeBps), 0, 0);
3697 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3698 // Frame should not be dropped.
3699 WaitForEncodedFrame(2);
3700
3701 video_stream_encoder_->OnBitrateUpdated(
3702 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 11:36:553703 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Åsa Persson139f4dc2019-08-02 07:29:583704 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
3705 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3706 // Expect to drop this frame, the wait should time out.
3707 ExpectDroppedFrame();
3708
3709 // Expect the sink_wants to specify a scaled frame.
3710 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3711 video_stream_encoder_->Stop();
3712}
3713
Åsa Perssone644a032019-11-08 14:56:003714TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
3715 webrtc::test::ScopedFieldTrials field_trials(
3716 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
3717
3718 // Reset encoder for field trials to take effect.
3719 VideoEncoderConfig config = video_encoder_config_.Copy();
3720 config.max_bitrate_bps = kTargetBitrateBps;
3721 ConfigureEncoder(std::move(config));
3722 fake_encoder_.SetQp(kQpLow);
3723
3724 // Enable MAINTAIN_FRAMERATE preference.
3725 AdaptingFrameForwarder source;
3726 source.set_adaptation_enabled(true);
3727 video_stream_encoder_->SetSource(&source,
3728 DegradationPreference::MAINTAIN_FRAMERATE);
3729
3730 // Start at low bitrate.
3731 const int kLowBitrateBps = 200000;
3732 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowBitrateBps),
3733 DataRate::bps(kLowBitrateBps),
3734 DataRate::bps(kLowBitrateBps), 0, 0);
3735
3736 // Expect first frame to be dropped and resolution to be limited.
3737 const int kWidth = 1280;
3738 const int kHeight = 720;
3739 const int64_t kFrameIntervalMs = 100;
3740 int64_t timestamp_ms = kFrameIntervalMs;
3741 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3742 ExpectDroppedFrame();
3743 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3744
3745 // Increase bitrate to encoder max.
3746 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(config.max_bitrate_bps),
3747 DataRate::bps(config.max_bitrate_bps),
3748 DataRate::bps(config.max_bitrate_bps),
3749 0, 0);
3750
3751 // Insert frames and advance |min_duration_ms|.
3752 for (size_t i = 1; i <= 10; i++) {
3753 timestamp_ms += kFrameIntervalMs;
3754 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3755 WaitForEncodedFrame(timestamp_ms);
3756 }
3757 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3758 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3759
3760 fake_clock_.AdvanceTime(TimeDelta::ms(2000));
3761
3762 // Insert frame should trigger high BW and release quality limitation.
3763 timestamp_ms += kFrameIntervalMs;
3764 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3765 WaitForEncodedFrame(timestamp_ms);
3766 VerifyFpsMaxResolutionMax(source.sink_wants());
3767
3768 // Frame should not be adapted.
3769 timestamp_ms += kFrameIntervalMs;
3770 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3771 WaitForEncodedFrame(kWidth, kHeight);
3772 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3773
3774 video_stream_encoder_->Stop();
3775}
3776
mflodmancc3d4422017-08-03 15:27:513777TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 08:47:313778 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
3779 const int kTooSmallWidth = 10;
3780 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 13:14:013781 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553782 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3783 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 08:47:313784
Taylor Brandstetter49fcc102018-05-16 21:20:413785 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 08:47:313786 test::FrameForwarder source;
mflodmancc3d4422017-08-03 15:27:513787 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413788 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 08:47:313789 VerifyNoLimitation(source.sink_wants());
3790 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3791
3792 // Trigger adapt down, too small frame, expect no change.
3793 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:523794 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:513795 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 08:42:193796 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 08:47:313797 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3798 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3799
mflodmancc3d4422017-08-03 15:27:513800 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 08:47:313801}
3802
mflodmancc3d4422017-08-03 15:27:513803TEST_F(VideoStreamEncoderTest,
3804 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:223805 const int kTooSmallWidth = 10;
3806 const int kTooSmallHeight = 10;
3807 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 13:14:013808 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553809 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3810 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-14 06:25:223811
Taylor Brandstetter49fcc102018-05-16 21:20:413812 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:223813 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 21:20:413814 video_stream_encoder_->SetSource(&source,
3815 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:223816 VerifyNoLimitation(source.sink_wants());
3817 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3818 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3819
3820 // Trigger adapt down, expect limited framerate.
3821 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:523822 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 15:27:513823 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223824 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3825 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3826 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3827 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3828
3829 // Trigger adapt down, too small frame, expect no change.
3830 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 14:06:523831 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 15:27:513832 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:223833 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3834 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3835 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3836 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3837
mflodmancc3d4422017-08-03 15:27:513838 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:223839}
3840
mflodmancc3d4422017-08-03 15:27:513841TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 08:12:523842 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 13:14:013843 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553844 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3845 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerf1338562018-04-26 07:51:473846 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 08:12:523847 const int kFrameWidth = 1280;
3848 const int kFrameHeight = 720;
3849 video_source_.IncomingCapturedFrame(
3850 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523851 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 15:27:513852 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 08:12:523853}
3854
sprangb1ca0732017-02-01 16:38:123855// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 15:27:513856TEST_F(VideoStreamEncoderTest,
3857 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 13:14:013858 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553859 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3860 DataRate::bps(kTargetBitrateBps), 0, 0);
sprangb1ca0732017-02-01 16:38:123861
3862 const int kFrameWidth = 1280;
3863 const int kFrameHeight = 720;
3864 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 15:27:513865 // requested by
3866 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 16:38:123867 video_source_.set_adaptation_enabled(true);
3868
3869 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:193870 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523871 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:123872
3873 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:513874 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 16:38:123875 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:193876 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523877 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 16:38:123878
asaperssonfab67072017-04-04 12:51:493879 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 15:27:513880 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 16:38:123881 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 08:42:193882 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523883 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 16:38:123884
mflodmancc3d4422017-08-03 15:27:513885 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 16:38:123886}
sprangfe627f32017-03-29 15:24:593887
mflodmancc3d4422017-08-03 15:27:513888TEST_F(VideoStreamEncoderTest,
3889 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-03 06:53:043890 const int kFrameWidth = 1280;
3891 const int kFrameHeight = 720;
sprangc5d62e22017-04-03 06:53:043892
Erik Språng4c6ca302019-04-08 13:14:013893 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553894 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3895 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 15:27:513896 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:413897 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:043898 video_source_.set_adaptation_enabled(true);
3899
sprang4847ae62017-06-27 14:06:523900 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-03 06:53:043901
3902 video_source_.IncomingCapturedFrame(
3903 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523904 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:043905
3906 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 15:27:513907 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043908
3909 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 14:06:523910 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:043911 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:043912 video_source_.IncomingCapturedFrame(
3913 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523914 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-03 06:53:043915 }
3916
3917 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 15:27:513918 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043919 int num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:523920 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:043921 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:043922 video_source_.IncomingCapturedFrame(
3923 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523924 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:043925 ++num_frames_dropped;
3926 } else {
Åsa Perssonc74d8da2017-12-04 13:13:563927 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:043928 }
3929 }
3930
sprang4847ae62017-06-27 14:06:523931 // Add some slack to account for frames dropped by the frame dropper.
3932 const int kErrorMargin = 1;
3933 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:043934 kErrorMargin);
3935
3936 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 15:27:513937 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-03 06:53:043938 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 08:42:193939 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:043940 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:043941 video_source_.IncomingCapturedFrame(
3942 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523943 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:043944 ++num_frames_dropped;
3945 } else {
Åsa Perssonc74d8da2017-12-04 13:13:563946 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:043947 }
3948 }
sprang4847ae62017-06-27 14:06:523949 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-03 06:53:043950 kErrorMargin);
3951
3952 // Go back up one step.
mflodmancc3d4422017-08-03 15:27:513953 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-03 06:53:043954 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:523955 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:043956 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:043957 video_source_.IncomingCapturedFrame(
3958 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523959 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:043960 ++num_frames_dropped;
3961 } else {
Åsa Perssonc74d8da2017-12-04 13:13:563962 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:043963 }
3964 }
sprang4847ae62017-06-27 14:06:523965 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-03 06:53:043966 kErrorMargin);
3967
3968 // Go back up to original mode.
mflodmancc3d4422017-08-03 15:27:513969 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-03 06:53:043970 num_frames_dropped = 0;
sprang4847ae62017-06-27 14:06:523971 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-03 06:53:043972 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-03 06:53:043973 video_source_.IncomingCapturedFrame(
3974 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:523975 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-03 06:53:043976 ++num_frames_dropped;
3977 } else {
Åsa Perssonc74d8da2017-12-04 13:13:563978 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-03 06:53:043979 }
3980 }
3981 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
3982
mflodmancc3d4422017-08-03 15:27:513983 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:043984}
3985
mflodmancc3d4422017-08-03 15:27:513986TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-03 06:53:043987 const int kFramerateFps = 5;
3988 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-03 06:53:043989 const int kFrameWidth = 1280;
3990 const int kFrameHeight = 720;
3991
sprang4847ae62017-06-27 14:06:523992 // Reconfigure encoder with two temporal layers and screensharing, which will
3993 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 07:51:473994 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 14:06:523995
Erik Språng4c6ca302019-04-08 13:14:013996 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:553997 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3998 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 15:27:513999 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 21:20:414000 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-03 06:53:044001 video_source_.set_adaptation_enabled(true);
4002
sprang4847ae62017-06-27 14:06:524003 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-03 06:53:044004
4005 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-09 01:04:294006 rtc::VideoSinkWants last_wants;
4007 do {
4008 last_wants = video_source_.sink_wants();
4009
sprangc5d62e22017-04-03 06:53:044010 // Insert frames to get a new fps estimate...
4011 for (int j = 0; j < kFramerateFps; ++j) {
4012 video_source_.IncomingCapturedFrame(
4013 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-09 01:04:294014 if (video_source_.last_sent_width()) {
4015 sink_.WaitForEncodedFrame(timestamp_ms);
4016 }
sprangc5d62e22017-04-03 06:53:044017 timestamp_ms += kFrameIntervalMs;
Sebastian Jansson40889f32019-04-17 10:11:204018 fake_clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalMs));
sprangc5d62e22017-04-03 06:53:044019 }
4020 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 15:27:514021 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-09 01:04:294022 } while (video_source_.sink_wants().max_framerate_fps <
4023 last_wants.max_framerate_fps);
sprangc5d62e22017-04-03 06:53:044024
Jonathan Yubc771b72017-12-09 01:04:294025 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-14 06:25:224026
mflodmancc3d4422017-08-03 15:27:514027 video_stream_encoder_->Stop();
sprangc5d62e22017-04-03 06:53:044028}
asaperssonf7e294d2017-06-14 06:25:224029
mflodmancc3d4422017-08-03 15:27:514030TEST_F(VideoStreamEncoderTest,
4031 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-14 06:25:224032 const int kWidth = 1280;
4033 const int kHeight = 720;
4034 const int64_t kFrameIntervalMs = 150;
4035 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 13:14:014036 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554037 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4038 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-14 06:25:224039
Taylor Brandstetter49fcc102018-05-16 21:20:414040 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:224041 AdaptingFrameForwarder source;
4042 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:414043 video_stream_encoder_->SetSource(&source,
4044 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:224045 timestamp_ms += kFrameIntervalMs;
4046 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524047 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:194048 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224049 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4050 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4051 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4052
4053 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:514054 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224055 timestamp_ms += kFrameIntervalMs;
4056 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524057 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224058 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4059 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4060 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4061 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4062
4063 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514064 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224065 timestamp_ms += kFrameIntervalMs;
4066 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524067 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224068 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4069 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4070 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4071 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4072
4073 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514074 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224075 timestamp_ms += kFrameIntervalMs;
4076 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524077 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224078 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4079 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4080 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4081 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4082
4083 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:514084 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224085 timestamp_ms += kFrameIntervalMs;
4086 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524087 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224088 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4089 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4090 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4091 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4092
4093 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:514094 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224095 timestamp_ms += kFrameIntervalMs;
4096 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524097 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224098 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4099 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4100 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4101 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4102
4103 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:514104 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224105 timestamp_ms += kFrameIntervalMs;
4106 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524107 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224108 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4109 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4110 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4111 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4112
4113 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 15:27:514114 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224115 timestamp_ms += kFrameIntervalMs;
4116 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524117 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224118 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4119 rtc::VideoSinkWants last_wants = source.sink_wants();
4120 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4121 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4122 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4123
4124 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 15:27:514125 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224126 timestamp_ms += kFrameIntervalMs;
4127 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524128 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224129 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4130 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4131 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4132 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4133
4134 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 15:27:514135 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224136 timestamp_ms += kFrameIntervalMs;
4137 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524138 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224139 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4140 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4141 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4142 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4143
4144 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 15:27:514145 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224146 timestamp_ms += kFrameIntervalMs;
4147 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524148 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224149 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4150 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4151 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4152 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4153
4154 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:514155 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224156 timestamp_ms += kFrameIntervalMs;
4157 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524158 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224159 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4160 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4161 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4162 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4163
4164 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514165 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224166 timestamp_ms += kFrameIntervalMs;
4167 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524168 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224169 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4170 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4171 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4172 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4173
4174 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514175 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224176 timestamp_ms += kFrameIntervalMs;
4177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524178 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224179 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4180 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4181 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4182 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4183
4184 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:514185 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224186 timestamp_ms += kFrameIntervalMs;
4187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524188 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224189 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4190 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4191 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4192 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4193
Åsa Persson30ab0152019-08-27 10:22:334194 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:514195 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224196 timestamp_ms += kFrameIntervalMs;
4197 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524198 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-14 06:25:224199 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 08:42:194200 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224201 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4202 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4203 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4204
4205 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:514206 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194207 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224208 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4209
mflodmancc3d4422017-08-03 15:27:514210 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224211}
4212
mflodmancc3d4422017-08-03 15:27:514213TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-14 06:25:224214 const int kWidth = 1280;
4215 const int kHeight = 720;
4216 const int64_t kFrameIntervalMs = 150;
4217 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 13:14:014218 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554219 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4220 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-14 06:25:224221
Taylor Brandstetter49fcc102018-05-16 21:20:414222 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:224223 AdaptingFrameForwarder source;
4224 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:414225 video_stream_encoder_->SetSource(&source,
4226 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:224227 timestamp_ms += kFrameIntervalMs;
4228 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524229 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:194230 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224231 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4232 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4233 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4234 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4235 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4236 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4237
4238 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:514239 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:224240 timestamp_ms += kFrameIntervalMs;
4241 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524242 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224243 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4245 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4246 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4247 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4248 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4249 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4250
4251 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514252 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:224253 timestamp_ms += kFrameIntervalMs;
4254 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524255 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224256 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4257 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4258 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4259 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4260 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4261 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4262 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4263
4264 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514265 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224266 timestamp_ms += kFrameIntervalMs;
4267 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524268 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224269 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4270 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4271 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4272 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4273 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4274 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4275 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4276
4277 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514278 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-14 06:25:224279 timestamp_ms += kFrameIntervalMs;
4280 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524281 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224282 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4283 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4284 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4285 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4286 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4287 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4288 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4289
4290 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 15:27:514291 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224292 timestamp_ms += kFrameIntervalMs;
4293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524294 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224295 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4296 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4298 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4299 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4300 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4301 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4302
4303 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 15:27:514304 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-14 06:25:224305 timestamp_ms += kFrameIntervalMs;
4306 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524307 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-14 06:25:224308 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 08:42:194309 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224310 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4311 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4312 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4313 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4314 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4315 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4316
4317 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:514318 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194319 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224320 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4321 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4322
mflodmancc3d4422017-08-03 15:27:514323 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224324}
4325
mflodmancc3d4422017-08-03 15:27:514326TEST_F(VideoStreamEncoderTest,
4327 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-14 06:25:224328 const int kWidth = 640;
4329 const int kHeight = 360;
4330 const int kFpsLimit = 15;
4331 const int64_t kFrameIntervalMs = 150;
4332 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 13:14:014333 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554334 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4335 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-14 06:25:224336
Taylor Brandstetter49fcc102018-05-16 21:20:414337 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-14 06:25:224338 AdaptingFrameForwarder source;
4339 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 21:20:414340 video_stream_encoder_->SetSource(&source,
4341 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-14 06:25:224342 timestamp_ms += kFrameIntervalMs;
4343 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524344 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 08:42:194345 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224346 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4347 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4348 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4349 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4350 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4351 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4352
4353 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514354 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-14 06:25:224355 timestamp_ms += kFrameIntervalMs;
4356 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524357 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224358 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4359 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4360 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4361 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4362 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4363 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4364 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4365
4366 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 15:27:514367 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-14 06:25:224368 timestamp_ms += kFrameIntervalMs;
4369 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524370 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224371 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4372 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4373 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4374 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4375 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4376 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4377 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4378
4379 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 15:27:514380 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-14 06:25:224381 timestamp_ms += kFrameIntervalMs;
4382 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524383 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-14 06:25:224384 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4385 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4386 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4387 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4388 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4389 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4390 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4391
4392 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 15:27:514393 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-14 06:25:224394 timestamp_ms += kFrameIntervalMs;
4395 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 14:06:524396 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 08:42:194397 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224398 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4399 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4400 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4401 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4402 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4403 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4404
4405 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 15:27:514406 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 08:42:194407 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-14 06:25:224408 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4409 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4410
mflodmancc3d4422017-08-03 15:27:514411 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-14 06:25:224412}
4413
mflodmancc3d4422017-08-03 15:27:514414TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 13:53:484415 const int kFrameWidth = 1920;
4416 const int kFrameHeight = 1080;
4417 // 3/4 of 1920.
4418 const int kAdaptedFrameWidth = 1440;
4419 // 3/4 of 1080 rounded down to multiple of 4.
4420 const int kAdaptedFrameHeight = 808;
4421 const int kFramerate = 24;
4422
Erik Språng4c6ca302019-04-08 13:14:014423 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554424 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4425 DataRate::bps(kTargetBitrateBps), 0, 0);
ilnik6b826ef2017-06-16 13:53:484426 // Trigger reconfigure encoder (without resetting the entire instance).
4427 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 13:36:514428 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 13:53:484429 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4430 video_encoder_config.number_of_streams = 1;
4431 video_encoder_config.video_stream_factory =
4432 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 15:27:514433 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 07:51:474434 kMaxPayloadLength);
mflodmancc3d4422017-08-03 15:27:514435 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 13:53:484436
4437 video_source_.set_adaptation_enabled(true);
4438
4439 video_source_.IncomingCapturedFrame(
4440 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524441 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 13:53:484442
4443 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 15:27:514444 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 13:53:484445 video_source_.IncomingCapturedFrame(
4446 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 14:06:524447 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 13:53:484448
mflodmancc3d4422017-08-03 15:27:514449 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 13:53:484450}
4451
mflodmancc3d4422017-08-03 15:27:514452TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 14:06:524453 const int kFrameWidth = 1280;
4454 const int kFrameHeight = 720;
4455 const int kLowFps = 2;
4456 const int kHighFps = 30;
4457
Erik Språng4c6ca302019-04-08 13:14:014458 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554459 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4460 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 14:06:524461
4462 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4463 max_framerate_ = kLowFps;
4464
4465 // Insert 2 seconds of 2fps video.
4466 for (int i = 0; i < kLowFps * 2; ++i) {
4467 video_source_.IncomingCapturedFrame(
4468 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4469 WaitForEncodedFrame(timestamp_ms);
4470 timestamp_ms += 1000 / kLowFps;
4471 }
4472
4473 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 13:14:014474 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554475 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4476 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 14:06:524477 video_source_.IncomingCapturedFrame(
4478 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4479 WaitForEncodedFrame(timestamp_ms);
4480 timestamp_ms += 1000 / kLowFps;
4481
4482 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4483
4484 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 08:48:484485 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 14:06:524486 const int kFrameIntervalMs = 1000 / kHighFps;
4487 max_framerate_ = kHighFps;
4488 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4489 video_source_.IncomingCapturedFrame(
4490 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4491 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4492 // be dropped if the encoder hans't been updated with the new higher target
4493 // framerate yet, causing it to overshoot the target bitrate and then
4494 // suffering the wrath of the media optimizer.
4495 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4496 timestamp_ms += kFrameIntervalMs;
4497 }
4498
4499 // Don expect correct measurement just yet, but it should be higher than
4500 // before.
4501 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4502
mflodmancc3d4422017-08-03 15:27:514503 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:524504}
4505
mflodmancc3d4422017-08-03 15:27:514506TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 14:06:524507 const int kFrameWidth = 1280;
4508 const int kFrameHeight = 720;
4509 const int kTargetBitrateBps = 1000000;
4510
4511 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 12:09:314512 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 13:14:014513 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554514 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4515 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 15:27:514516 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 14:06:524517
4518 // Insert a first video frame, causes another bitrate update.
4519 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4520 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4521 video_source_.IncomingCapturedFrame(
4522 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4523 WaitForEncodedFrame(timestamp_ms);
4524
4525 // Next, simulate video suspension due to pacer queue overrun.
Florent Castellia8336d32019-09-09 11:36:554526 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0),
4527 DataRate::bps(0), 0, 1);
sprang4847ae62017-06-27 14:06:524528
4529 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 08:48:484530 timestamp_ms += kProcessIntervalMs;
4531 fake_clock_.AdvanceTime(TimeDelta::ms(kProcessIntervalMs));
sprang4847ae62017-06-27 14:06:524532
4533 // Bitrate observer should not be called.
4534 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4535 video_source_.IncomingCapturedFrame(
4536 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4537 ExpectDroppedFrame();
4538
mflodmancc3d4422017-08-03 15:27:514539 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 14:06:524540}
ilnik6b826ef2017-06-16 13:53:484541
Niels Möller4db138e2018-04-19 07:04:134542TEST_F(VideoStreamEncoderTest,
4543 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4544 const int kFrameWidth = 1280;
4545 const int kFrameHeight = 720;
4546 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 13:14:014547 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554548 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4549 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 07:04:134550 video_source_.IncomingCapturedFrame(
4551 CreateFrame(1, kFrameWidth, kFrameHeight));
4552 WaitForEncodedFrame(1);
4553 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4554 .low_encode_usage_threshold_percent,
4555 default_options.low_encode_usage_threshold_percent);
4556 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4557 .high_encode_usage_threshold_percent,
4558 default_options.high_encode_usage_threshold_percent);
4559 video_stream_encoder_->Stop();
4560}
4561
4562TEST_F(VideoStreamEncoderTest,
4563 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4564 const int kFrameWidth = 1280;
4565 const int kFrameHeight = 720;
4566 CpuOveruseOptions hardware_options;
4567 hardware_options.low_encode_usage_threshold_percent = 150;
4568 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 11:42:184569 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 07:04:134570
Erik Språng4c6ca302019-04-08 13:14:014571 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554572 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4573 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 07:04:134574 video_source_.IncomingCapturedFrame(
4575 CreateFrame(1, kFrameWidth, kFrameHeight));
4576 WaitForEncodedFrame(1);
4577 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4578 .low_encode_usage_threshold_percent,
4579 hardware_options.low_encode_usage_threshold_percent);
4580 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4581 .high_encode_usage_threshold_percent,
4582 hardware_options.high_encode_usage_threshold_percent);
4583 video_stream_encoder_->Stop();
4584}
4585
Niels Möller6bb5ab92019-01-11 10:11:104586TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4587 const int kFrameWidth = 320;
4588 const int kFrameHeight = 240;
4589 const int kFps = 30;
4590 const int kTargetBitrateBps = 120000;
4591 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4592
Erik Språng4c6ca302019-04-08 13:14:014593 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554594 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4595 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:104596
4597 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4598 max_framerate_ = kFps;
4599
4600 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4601 fake_encoder_.SimulateOvershoot(1.0);
4602 int num_dropped = 0;
4603 for (int i = 0; i < kNumFramesInRun; ++i) {
4604 video_source_.IncomingCapturedFrame(
4605 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4606 // Wait up to two frame durations for a frame to arrive.
4607 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4608 ++num_dropped;
4609 }
4610 timestamp_ms += 1000 / kFps;
4611 }
4612
Erik Språnga8d48ab2019-02-08 13:17:404613 // Framerate should be measured to be near the expected target rate.
4614 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4615
4616 // Frame drops should be within 5% of expected 0%.
4617 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:104618
4619 // Make encoder produce frames at double the expected bitrate during 3 seconds
4620 // of video, verify number of drops. Rate needs to be slightly changed in
4621 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 15:20:174622 double overshoot_factor = 2.0;
4623 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4624 // With bitrate adjuster, when need to overshoot even more to trigger
4625 // frame dropping.
4626 overshoot_factor *= 2;
4627 }
4628 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 14:37:334629 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 13:14:014630 DataRate::bps(kTargetBitrateBps + 1000),
Florent Castellia8336d32019-09-09 11:36:554631 DataRate::bps(kTargetBitrateBps + 1000),
Erik Språng4c6ca302019-04-08 13:14:014632 DataRate::bps(kTargetBitrateBps + 1000), 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:104633 num_dropped = 0;
4634 for (int i = 0; i < kNumFramesInRun; ++i) {
4635 video_source_.IncomingCapturedFrame(
4636 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4637 // Wait up to two frame durations for a frame to arrive.
4638 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4639 ++num_dropped;
4640 }
4641 timestamp_ms += 1000 / kFps;
4642 }
4643
Erik Språng4c6ca302019-04-08 13:14:014644 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554645 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4646 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språnga8d48ab2019-02-08 13:17:404647
4648 // Target framerate should be still be near the expected target, despite
4649 // the frame drops.
4650 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4651
4652 // Frame drops should be within 5% of expected 50%.
4653 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 10:11:104654
4655 video_stream_encoder_->Stop();
4656}
4657
4658TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4659 const int kFrameWidth = 320;
4660 const int kFrameHeight = 240;
4661 const int kActualInputFps = 24;
4662 const int kTargetBitrateBps = 120000;
4663
4664 ASSERT_GT(max_framerate_, kActualInputFps);
4665
4666 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4667 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 13:14:014668 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554669 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4670 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 10:11:104671
4672 // Insert 3 seconds of video, with an input fps lower than configured max.
4673 for (int i = 0; i < kActualInputFps * 3; ++i) {
4674 video_source_.IncomingCapturedFrame(
4675 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4676 // Wait up to two frame durations for a frame to arrive.
4677 WaitForEncodedFrame(timestamp_ms);
4678 timestamp_ms += 1000 / kActualInputFps;
4679 }
4680
4681 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4682
4683 video_stream_encoder_->Stop();
4684}
4685
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:264686TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4687 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 13:14:014688 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554689 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4690 DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:264691
4692 fake_encoder_.BlockNextEncode();
4693 video_source_.IncomingCapturedFrame(
4694 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4695 WaitForEncodedFrame(1);
4696 // On the very first frame full update should be forced.
4697 rect = fake_encoder_.GetLastUpdateRect();
4698 EXPECT_EQ(rect.offset_x, 0);
4699 EXPECT_EQ(rect.offset_y, 0);
4700 EXPECT_EQ(rect.height, codec_height_);
4701 EXPECT_EQ(rect.width, codec_width_);
4702 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4703 // call to ContinueEncode.
4704 video_source_.IncomingCapturedFrame(
4705 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4706 ExpectDroppedFrame();
4707 video_source_.IncomingCapturedFrame(
4708 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4709 ExpectDroppedFrame();
4710 fake_encoder_.ContinueEncode();
4711 WaitForEncodedFrame(3);
4712 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4713 rect = fake_encoder_.GetLastUpdateRect();
4714 EXPECT_EQ(rect.offset_x, 1);
4715 EXPECT_EQ(rect.offset_y, 0);
4716 EXPECT_EQ(rect.width, 10);
4717 EXPECT_EQ(rect.height, 1);
4718
4719 video_source_.IncomingCapturedFrame(
4720 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4721 WaitForEncodedFrame(4);
4722 // Previous frame was encoded, so no accumulation should happen.
4723 rect = fake_encoder_.GetLastUpdateRect();
4724 EXPECT_EQ(rect.offset_x, 0);
4725 EXPECT_EQ(rect.offset_y, 0);
4726 EXPECT_EQ(rect.width, 1);
4727 EXPECT_EQ(rect.height, 1);
4728
4729 video_stream_encoder_->Stop();
4730}
4731
Erik Språngd7329ca2019-02-21 20:19:534732TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 13:14:014733 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554734 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4735 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 20:19:534736
4737 // First frame is always keyframe.
4738 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4739 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 14:43:584740 EXPECT_THAT(
4741 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124742 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:534743
4744 // Insert delta frame.
4745 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4746 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 14:43:584747 EXPECT_THAT(
4748 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124749 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:534750
4751 // Request next frame be a key-frame.
4752 video_stream_encoder_->SendKeyFrame();
4753 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4754 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 14:43:584755 EXPECT_THAT(
4756 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124757 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:534758
4759 video_stream_encoder_->Stop();
4760}
4761
4762TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
4763 // Setup simulcast with three streams.
4764 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 14:37:334765 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 13:14:014766 DataRate::bps(kSimulcastTargetBitrateBps),
Florent Castellia8336d32019-09-09 11:36:554767 DataRate::bps(kSimulcastTargetBitrateBps),
Erik Språng4c6ca302019-04-08 13:14:014768 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 20:19:534769 // Wait for all three layers before triggering event.
4770 sink_.SetNumExpectedLayers(3);
4771
4772 // First frame is always keyframe.
4773 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4774 WaitForEncodedFrame(1);
4775 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124776 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
4777 VideoFrameType::kVideoFrameKey,
4778 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:534779
4780 // Insert delta frame.
4781 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4782 WaitForEncodedFrame(2);
4783 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124784 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
4785 VideoFrameType::kVideoFrameDelta,
4786 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:534787
4788 // Request next frame be a key-frame.
4789 // Only first stream is configured to produce key-frame.
4790 video_stream_encoder_->SendKeyFrame();
4791 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4792 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 11:45:394793
4794 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
4795 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 20:19:534796 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124797 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 11:45:394798 VideoFrameType::kVideoFrameKey,
4799 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:534800
4801 video_stream_encoder_->Stop();
4802}
4803
4804TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
4805 // Configure internal source factory and setup test again.
4806 encoder_factory_.SetHasInternalSource(true);
4807 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 13:14:014808 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554809 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4810 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 20:19:534811
4812 // Call encoder directly, simulating internal source where encoded frame
4813 // callback in VideoStreamEncoder is called despite no OnFrame().
4814 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
4815 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:584816 EXPECT_THAT(
4817 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124818 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:534819
Niels Möller8f7ce222019-03-21 14:43:584820 const std::vector<VideoFrameType> kDeltaFrame = {
4821 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 20:19:534822 // Need to set timestamp manually since manually for injected frame.
4823 VideoFrame frame = CreateFrame(101, nullptr);
4824 frame.set_timestamp(101);
4825 fake_encoder_.InjectFrame(frame, false);
4826 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:584827 EXPECT_THAT(
4828 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124829 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 20:19:534830
4831 // Request key-frame. The forces a dummy frame down into the encoder.
4832 fake_encoder_.ExpectNullFrame();
4833 video_stream_encoder_->SendKeyFrame();
4834 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 14:43:584835 EXPECT_THAT(
4836 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 13:11:124837 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 20:19:534838
4839 video_stream_encoder_->Stop();
4840}
Erik Språngb7cb7b52019-02-26 14:52:334841
4842TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
4843 // Configure internal source factory and setup test again.
4844 encoder_factory_.SetHasInternalSource(true);
4845 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 13:14:014846 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554847 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4848 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngb7cb7b52019-02-26 14:52:334849
4850 int64_t timestamp = 1;
4851 EncodedImage image;
Niels Möller4d504c72019-06-18 13:56:564852 image.SetEncodedData(
4853 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 14:52:334854 image.capture_time_ms_ = ++timestamp;
4855 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
4856 const int64_t kEncodeFinishDelayMs = 10;
4857 image.timing_.encode_start_ms = timestamp;
4858 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
4859 fake_encoder_.InjectEncodedImage(image);
4860 // Wait for frame without incrementing clock.
4861 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4862 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
4863 // capture timestamp should be kEncodeFinishDelayMs in the past.
4864 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
4865 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
4866 kEncodeFinishDelayMs);
4867
4868 video_stream_encoder_->Stop();
4869}
Mirta Dvornicic28f0eb22019-05-28 14:30:164870
4871TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
4872 // Configure internal source factory and setup test again.
4873 encoder_factory_.SetHasInternalSource(true);
4874 ResetEncoder("H264", 1, 1, 1, false);
4875
4876 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
4877 image._frameType = VideoFrameType::kVideoFrameKey;
4878
4879 CodecSpecificInfo codec_specific_info;
4880 codec_specific_info.codecType = kVideoCodecH264;
4881
4882 RTPFragmentationHeader fragmentation;
4883 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4884 fragmentation.fragmentationOffset[0] = 4;
4885 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
4886
4887 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4888 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4889
4890 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4891 testing::ElementsAreArray(optimal_sps));
4892 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4893 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4894 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4895 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4896
4897 video_stream_encoder_->Stop();
4898}
4899
4900TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
4901 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
4902 0x00, 0x00, 0x03, 0x03, 0xF4,
4903 0x05, 0x03, 0xC7, 0xC0};
4904
4905 // Configure internal source factory and setup test again.
4906 encoder_factory_.SetHasInternalSource(true);
4907 ResetEncoder("H264", 1, 1, 1, false);
4908
4909 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
4910 image._frameType = VideoFrameType::kVideoFrameKey;
4911
4912 CodecSpecificInfo codec_specific_info;
4913 codec_specific_info.codecType = kVideoCodecH264;
4914
4915 RTPFragmentationHeader fragmentation;
4916 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4917 fragmentation.fragmentationOffset[0] = 4;
4918 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
4919
4920 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4921 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4922
4923 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4924 testing::ElementsAreArray(optimal_sps));
4925 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4926 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4927 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4928 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4929
4930 video_stream_encoder_->Stop();
4931}
4932
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:144933TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
4934 const int kFrameWidth = 1280;
4935 const int kFrameHeight = 720;
4936 const int kTargetBitrateBps = 300000; // To low for HD resolution.
4937
4938 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:554939 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4940 DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:144941 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4942
4943 // Insert a first video frame. It should be dropped because of downscale in
4944 // resolution.
4945 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4946 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4947 frame.set_rotation(kVideoRotation_270);
4948 video_source_.IncomingCapturedFrame(frame);
4949
4950 ExpectDroppedFrame();
4951
4952 // Second frame is downscaled.
4953 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4954 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4955 frame.set_rotation(kVideoRotation_90);
4956 video_source_.IncomingCapturedFrame(frame);
4957
4958 WaitForEncodedFrame(timestamp_ms);
4959 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
4960
4961 // Insert another frame, also downscaled.
4962 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4963 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4964 frame.set_rotation(kVideoRotation_180);
4965 video_source_.IncomingCapturedFrame(frame);
4966
4967 WaitForEncodedFrame(timestamp_ms);
4968 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
4969
4970 video_stream_encoder_->Stop();
4971}
4972
Erik Språng5056af02019-09-02 13:53:114973TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
4974 const int kFrameWidth = 320;
4975 const int kFrameHeight = 180;
4976
4977 // Initial rate.
4978 video_stream_encoder_->OnBitrateUpdated(
4979 /*target_bitrate=*/DataRate::kbps(300),
Florent Castellia8336d32019-09-09 11:36:554980 /*stable_target_bitrate=*/DataRate::kbps(300),
Erik Språng5056af02019-09-02 13:53:114981 /*link_allocation=*/DataRate::kbps(300),
4982 /*fraction_lost=*/0,
4983 /*rtt_ms=*/0);
4984
4985 // Insert a first video frame so that encoder gets configured.
4986 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4987 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4988 frame.set_rotation(kVideoRotation_270);
4989 video_source_.IncomingCapturedFrame(frame);
4990 WaitForEncodedFrame(timestamp_ms);
4991
4992 // Set a target rate below the minimum allowed by the codec settings.
4993 VideoCodec codec_config = fake_encoder_.codec_config();
4994 DataRate min_rate = DataRate::kbps(codec_config.minBitrate);
4995 DataRate target_rate = min_rate - DataRate::kbps(1);
4996 video_stream_encoder_->OnBitrateUpdated(
4997 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 11:36:554998 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 13:53:114999 /*link_allocation=*/target_rate,
5000 /*fraction_lost=*/0,
5001 /*rtt_ms=*/0);
5002 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5003
5004 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5005 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5006 ASSERT_TRUE(rate_settings.has_value());
5007 DataRate allocation_sum = DataRate::bps(rate_settings->bitrate.get_sum_bps());
5008 EXPECT_EQ(min_rate, allocation_sum);
5009 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5010
5011 video_stream_encoder_->Stop();
5012}
5013
Evan Shrubsolee32ae4f2019-09-25 10:50:235014TEST_F(VideoStreamEncoderTest, EncoderRatesPropegatedOnReconfigure) {
5015 video_stream_encoder_->OnBitrateUpdated(
5016 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
5017 DataRate::bps(kTargetBitrateBps), 0, 0);
5018 // Capture a frame and wait for it to synchronize with the encoder thread.
5019 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5020 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5021 WaitForEncodedFrame(1);
5022
5023 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5024 ASSERT_TRUE(prev_rate_settings.has_value());
5025 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5026 kDefaultFramerate);
5027
5028 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5029 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5030 timestamp_ms += 1000 / kDefaultFramerate;
5031 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5032 WaitForEncodedFrame(timestamp_ms);
5033 }
5034 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5035 kDefaultFramerate);
5036 // Capture larger frame to trigger a reconfigure.
5037 codec_height_ *= 2;
5038 codec_width_ *= 2;
5039 timestamp_ms += 1000 / kDefaultFramerate;
5040 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5041 WaitForEncodedFrame(timestamp_ms);
5042
5043 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5044 auto current_rate_settings =
5045 fake_encoder_.GetAndResetLastRateControlSettings();
5046 // Ensure we have actually reconfigured twice
5047 // The rate settings should have been set again even though
5048 // they haven't changed.
5049 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 07:55:035050 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 10:50:235051
5052 video_stream_encoder_->Stop();
5053}
5054
philipeld9cc8c02019-09-16 12:53:405055struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
5056 MOCK_METHOD0(RequestEncoderFallback, void());
5057 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
5058};
5059
5060TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5061 constexpr int kDontCare = 100;
5062
5063 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5064 video_send_config_.encoder_settings.encoder_switch_request_callback =
5065 &switch_callback;
5066 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5067 encoder_config.codec_type = kVideoCodecVP8;
5068 webrtc::test::ScopedFieldTrials field_trial(
5069 "WebRTC-NetworkCondition-EncoderSwitch/"
5070 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5071 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5072
5073 // Reset encoder for new configuration to take effect.
5074 ConfigureEncoder(std::move(encoder_config));
5075
5076 // Send one frame to trigger ReconfigureEncoder.
5077 video_source_.IncomingCapturedFrame(
5078 CreateFrame(kDontCare, kDontCare, kDontCare));
5079
5080 using Config = EncoderSwitchRequestCallback::Config;
5081 EXPECT_CALL(switch_callback,
5082 RequestEncoderSwitch(AllOf(Field(&Config::codec_name, "AV1"),
5083 Field(&Config::param, "ping"),
5084 Field(&Config::value, "pong"))));
5085
5086 video_stream_encoder_->OnBitrateUpdated(
5087 /*target_bitrate=*/DataRate::kbps(50),
5088 /*stable_target_bitrate=*/DataRate::kbps(kDontCare),
5089 /*link_allocation=*/DataRate::kbps(kDontCare),
5090 /*fraction_lost=*/0,
5091 /*rtt_ms=*/0);
5092
5093 video_stream_encoder_->Stop();
5094}
5095
5096TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5097 constexpr int kSufficientBitrateToNotDrop = 1000;
5098 constexpr int kHighRes = 500;
5099 constexpr int kLowRes = 100;
5100
5101 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5102 video_send_config_.encoder_settings.encoder_switch_request_callback =
5103 &switch_callback;
5104 webrtc::test::ScopedFieldTrials field_trial(
5105 "WebRTC-NetworkCondition-EncoderSwitch/"
5106 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5107 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5108 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5109 encoder_config.codec_type = kVideoCodecH264;
5110
5111 // Reset encoder for new configuration to take effect.
5112 ConfigureEncoder(std::move(encoder_config));
5113
5114 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5115 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5116 // not fail.
5117 video_stream_encoder_->OnBitrateUpdated(
5118 /*target_bitrate=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5119 /*stable_target_bitrate=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5120 /*link_allocation=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5121 /*fraction_lost=*/0,
5122 /*rtt_ms=*/0);
5123
5124 // Send one frame to trigger ReconfigureEncoder.
5125 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5126 WaitForEncodedFrame(1);
5127
5128 using Config = EncoderSwitchRequestCallback::Config;
5129 EXPECT_CALL(switch_callback,
5130 RequestEncoderSwitch(AllOf(Field(&Config::codec_name, "AV1"),
5131 Field(&Config::param, "ping"),
5132 Field(&Config::value, "pong"))));
5133
5134 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5135 WaitForEncodedFrame(2);
5136
5137 video_stream_encoder_->Stop();
5138}
5139
Evan Shrubsole7c079f62019-09-26 07:55:035140TEST_F(VideoStreamEncoderTest,
5141 AllocationPropegratedToEncoderWhenTargetRateChanged) {
5142 const int kFrameWidth = 320;
5143 const int kFrameHeight = 180;
5144
5145 // Set initial rate.
5146 auto rate = DataRate::kbps(100);
5147 video_stream_encoder_->OnBitrateUpdated(
5148 /*target_bitrate=*/rate,
5149 /*stable_target_bitrate=*/rate,
5150 /*link_allocation=*/rate,
5151 /*fraction_lost=*/0,
5152 /*rtt_ms=*/0);
5153
5154 // Insert a first video frame so that encoder gets configured.
5155 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5156 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5157 frame.set_rotation(kVideoRotation_270);
5158 video_source_.IncomingCapturedFrame(frame);
5159 WaitForEncodedFrame(timestamp_ms);
5160 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5161
5162 // Change of target bitrate propagates to the encoder.
5163 auto new_stable_rate = rate - DataRate::kbps(5);
5164 video_stream_encoder_->OnBitrateUpdated(
5165 /*target_bitrate=*/new_stable_rate,
5166 /*stable_target_bitrate=*/new_stable_rate,
5167 /*link_allocation=*/rate,
5168 /*fraction_lost=*/0,
5169 /*rtt_ms=*/0);
5170 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5171 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5172 video_stream_encoder_->Stop();
5173}
5174
5175TEST_F(VideoStreamEncoderTest,
5176 AllocationNotPropegratedToEncoderWhenTargetRateUnchanged) {
5177 const int kFrameWidth = 320;
5178 const int kFrameHeight = 180;
5179
5180 // Set initial rate.
5181 auto rate = DataRate::kbps(100);
5182 video_stream_encoder_->OnBitrateUpdated(
5183 /*target_bitrate=*/rate,
5184 /*stable_target_bitrate=*/rate,
5185 /*link_allocation=*/rate,
5186 /*fraction_lost=*/0,
5187 /*rtt_ms=*/0);
5188
5189 // Insert a first video frame so that encoder gets configured.
5190 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5191 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5192 frame.set_rotation(kVideoRotation_270);
5193 video_source_.IncomingCapturedFrame(frame);
5194 WaitForEncodedFrame(timestamp_ms);
5195 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5196
5197 // Set a higher target rate without changing the link_allocation. Should not
5198 // reset encoder's rate.
5199 auto new_stable_rate = rate - DataRate::kbps(5);
5200 video_stream_encoder_->OnBitrateUpdated(
5201 /*target_bitrate=*/rate,
5202 /*stable_target_bitrate=*/new_stable_rate,
5203 /*link_allocation=*/rate,
5204 /*fraction_lost=*/0,
5205 /*rtt_ms=*/0);
5206 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5207 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5208 video_stream_encoder_->Stop();
5209}
5210
Ilya Nikolaevskiy648b9d72019-12-03 15:54:175211TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5212 test::ScopedFieldTrials field_trials(
5213 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5214 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5215 const int kFramerateFps = 30;
5216 const int kWidth = 1920;
5217 const int kHeight = 1080;
5218 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5219 // Works on screenshare mode.
5220 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5221 // We rely on the automatic resolution adaptation, but we handle framerate
5222 // adaptation manually by mocking the stats proxy.
5223 video_source_.set_adaptation_enabled(true);
5224
5225 // BALANCED degradation preference is required for this feature.
5226 video_stream_encoder_->OnBitrateUpdated(
5227 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
5228 DataRate::bps(kTargetBitrateBps), 0, 0);
5229 video_stream_encoder_->SetSource(&video_source_,
5230 webrtc::DegradationPreference::BALANCED);
5231 VerifyNoLimitation(video_source_.sink_wants());
5232
5233 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5234 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5235
5236 // Pass enough frames with the full update to trigger animation detection.
5237 for (int i = 0; i < kNumFrames; ++i) {
5238 int64_t timestamp_ms =
5239 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5240 frame.set_ntp_time_ms(timestamp_ms);
5241 frame.set_timestamp_us(timestamp_ms * 1000);
5242 video_source_.IncomingCapturedFrame(frame);
5243 WaitForEncodedFrame(timestamp_ms);
5244 }
5245
5246 // Resolution should be limited.
5247 rtc::VideoSinkWants expected;
5248 expected.max_framerate_fps = kFramerateFps;
5249 expected.max_pixel_count = 1280 * 720 + 1;
5250 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5251
5252 // Pass one frame with no known update.
5253 // Resolution cap should be removed immediately.
5254 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5255 frame.set_ntp_time_ms(timestamp_ms);
5256 frame.set_timestamp_us(timestamp_ms * 1000);
5257 frame.clear_update_rect();
5258
5259 video_source_.IncomingCapturedFrame(frame);
5260 WaitForEncodedFrame(timestamp_ms);
5261
5262 // Resolution should be unlimited now.
5263 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5264
5265 video_stream_encoder_->Stop();
5266}
5267
perkj26091b12016-09-01 08:17:405268} // namespace webrtc