blob: 1b88bb35c802f5d8ddc6f25d24e8009b48e4dc94 [file] [log] [blame]
Henrik Boströmb08882b2020-01-07 09:11:171/*
2 * Copyright 2019 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "video/overuse_frame_detector_resource_adaptation_module.h"
12
13#include <algorithm>
14#include <limits>
15#include <memory>
16#include <string>
17#include <utility>
18
19#include "absl/algorithm/container.h"
Danil Chapovalov64f1f3f2020-01-16 13:41:1020#include "absl/base/macros.h"
Henrik Boström07b17df2020-01-15 10:42:1221#include "api/task_queue/task_queue_base.h"
Henrik Boströmb08882b2020-01-07 09:11:1722#include "api/video/video_source_interface.h"
Henrik Boström48258ac2020-02-06 11:49:5723#include "call/adaptation/resource.h"
Henrik Boströmce0ea492020-01-13 10:27:1824#include "call/adaptation/video_source_restrictions.h"
Henrik Boströmb08882b2020-01-07 09:11:1725#include "rtc_base/logging.h"
Henrik Boströmd2382002020-01-10 14:44:0126#include "rtc_base/numerics/safe_conversions.h"
Henrik Boströmb08882b2020-01-07 09:11:1727#include "rtc_base/strings/string_builder.h"
Henrik Boströmad515a22020-01-27 12:38:0528#include "rtc_base/time_utils.h"
Henrik Boströmb08882b2020-01-07 09:11:1729#include "video/video_stream_encoder.h"
30
31namespace webrtc {
32
33namespace {
34
35const int kMinFramerateFps = 2;
36
37bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
38 return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
39 degradation_preference == DegradationPreference::BALANCED;
40}
41
42bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
43 return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION ||
44 degradation_preference == DegradationPreference::BALANCED;
45}
46
Henrik Boström8234b922020-01-13 16:26:5047// Returns modified restrictions where any constraints that don't apply to the
48// degradation preference are cleared.
49VideoSourceRestrictions ApplyDegradationPreference(
50 VideoSourceRestrictions source_restrictions,
51 DegradationPreference degradation_preference) {
52 switch (degradation_preference) {
53 case DegradationPreference::BALANCED:
54 break;
55 case DegradationPreference::MAINTAIN_FRAMERATE:
56 source_restrictions.set_max_frame_rate(absl::nullopt);
57 break;
58 case DegradationPreference::MAINTAIN_RESOLUTION:
59 source_restrictions.set_max_pixels_per_frame(absl::nullopt);
60 source_restrictions.set_target_pixels_per_frame(absl::nullopt);
61 break;
62 case DegradationPreference::DISABLED:
63 source_restrictions.set_max_pixels_per_frame(absl::nullopt);
64 source_restrictions.set_target_pixels_per_frame(absl::nullopt);
65 source_restrictions.set_max_frame_rate(absl::nullopt);
66 }
67 return source_restrictions;
68}
69
Henrik Boströmb08882b2020-01-07 09:11:1770} // namespace
71
Henrik Boströmce0ea492020-01-13 10:27:1872// VideoSourceRestrictor is responsible for keeping track of current
73// VideoSourceRestrictions and how to modify them in response to adapting up or
74// down. It is not reponsible for determining when we should adapt up or down -
Henrik Boström7875c992020-02-06 09:35:0075// for that, see
76// OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse() and
77// OnResourceOveruse() - only how to modify the source/sink restrictions when
78// this happens. Note that it is also not responsible for reconfigruring the
Henrik Boströmce0ea492020-01-13 10:27:1879// source/sink, it is only a keeper of desired restrictions.
Henrik Boströmce0ea492020-01-13 10:27:1880class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
Henrik Boströmb08882b2020-01-07 09:11:1781 public:
Henrik Boström02956fe2020-02-25 08:35:0382 // For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
83 static int GetLowerFrameRateThan(int fps) {
84 RTC_DCHECK(fps != std::numeric_limits<int>::max());
85 return (fps * 2) / 3;
86 }
87 // TODO(hbos): Use absl::optional<> instead?
88 static int GetHigherFrameRateThan(int fps) {
89 return fps != std::numeric_limits<int>::max()
90 ? (fps * 3) / 2
91 : std::numeric_limits<int>::max();
92 }
93
94 // For resolution, the steps we take are 3/5 (down) and 5/3 (up).
95 // Notice the asymmetry of which restriction property is set depending on if
96 // we are adapting up or down:
97 // - DecreaseResolution() sets the max_pixels_per_frame() to the desired
98 // target and target_pixels_per_frame() to null.
99 // - IncreaseResolutionTo() sets the target_pixels_per_frame() to the desired
100 // target, and max_pixels_per_frame() is set according to
101 // GetIncreasedMaxPixelsWanted().
102 static int GetLowerResolutionThan(int pixel_count) {
103 RTC_DCHECK(pixel_count != std::numeric_limits<int>::max());
104 return (pixel_count * 3) / 5;
105 }
106 // TODO(hbos): Use absl::optional<> instead?
107 static int GetHigherResolutionThan(int pixel_count) {
108 return pixel_count != std::numeric_limits<int>::max()
109 ? (pixel_count * 5) / 3
110 : std::numeric_limits<int>::max();
111 }
112
Henrik Boströma3d42522020-01-16 12:55:29113 VideoSourceRestrictor() {}
Henrik Boströmb08882b2020-01-07 09:11:17114
Henrik Boströmce0ea492020-01-13 10:27:18115 VideoSourceRestrictions source_restrictions() {
Henrik Boströmce0ea492020-01-13 10:27:18116 return source_restrictions_;
Henrik Boströmd2382002020-01-10 14:44:01117 }
Henrik Boström8234b922020-01-13 16:26:50118 void ClearRestrictions() {
Henrik Boströmce0ea492020-01-13 10:27:18119 source_restrictions_ = VideoSourceRestrictions();
Henrik Boströmb08882b2020-01-07 09:11:17120 }
121
Henrik Boström02956fe2020-02-25 08:35:03122 bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) {
123 int max_pixels_per_frame = rtc::dchecked_cast<int>(
124 source_restrictions_.max_pixels_per_frame().value_or(
125 std::numeric_limits<int>::max()));
126 return target_pixels < max_pixels_per_frame &&
127 target_pixels >= min_pixels_per_frame;
128 }
129 void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) {
130 RTC_DCHECK(CanDecreaseResolutionTo(target_pixels, min_pixels_per_frame));
Henrik Boströmb08882b2020-01-07 09:11:17131 RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: "
Henrik Boström02956fe2020-02-25 08:35:03132 << target_pixels;
Henrik Boströmce0ea492020-01-13 10:27:18133 source_restrictions_.set_max_pixels_per_frame(
Henrik Boström02956fe2020-02-25 08:35:03134 target_pixels != std::numeric_limits<int>::max()
135 ? absl::optional<size_t>(target_pixels)
Henrik Boströmce0ea492020-01-13 10:27:18136 : absl::nullopt);
137 source_restrictions_.set_target_pixels_per_frame(absl::nullopt);
Henrik Boströmb08882b2020-01-07 09:11:17138 }
139
Henrik Boström02956fe2020-02-25 08:35:03140 bool CanIncreaseResolutionTo(int target_pixels) {
141 int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
142 int max_pixels_per_frame = rtc::dchecked_cast<int>(
143 source_restrictions_.max_pixels_per_frame().value_or(
144 std::numeric_limits<int>::max()));
145 return max_pixels_wanted > max_pixels_per_frame;
Henrik Boströmb08882b2020-01-07 09:11:17146 }
Henrik Boström02956fe2020-02-25 08:35:03147 void IncreaseResolutionTo(int target_pixels) {
148 RTC_DCHECK(CanIncreaseResolutionTo(target_pixels));
149 int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
Henrik Boströmb08882b2020-01-07 09:11:17150 RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
151 << max_pixels_wanted;
Henrik Boströmce0ea492020-01-13 10:27:18152 source_restrictions_.set_max_pixels_per_frame(
153 max_pixels_wanted != std::numeric_limits<int>::max()
154 ? absl::optional<size_t>(max_pixels_wanted)
155 : absl::nullopt);
156 source_restrictions_.set_target_pixels_per_frame(
157 max_pixels_wanted != std::numeric_limits<int>::max()
Henrik Boström02956fe2020-02-25 08:35:03158 ? absl::optional<size_t>(target_pixels)
Henrik Boströmce0ea492020-01-13 10:27:18159 : absl::nullopt);
Henrik Boströmb08882b2020-01-07 09:11:17160 }
161
Henrik Boström02956fe2020-02-25 08:35:03162 bool CanDecreaseFrameRateTo(int max_frame_rate) {
163 const int fps_wanted = std::max(kMinFramerateFps, max_frame_rate);
164 return fps_wanted < rtc::dchecked_cast<int>(
165 source_restrictions_.max_frame_rate().value_or(
166 std::numeric_limits<int>::max()));
Henrik Boströmb08882b2020-01-07 09:11:17167 }
Henrik Boström02956fe2020-02-25 08:35:03168 void DecreaseFrameRateTo(int max_frame_rate) {
169 RTC_DCHECK(CanDecreaseFrameRateTo(max_frame_rate));
170 max_frame_rate = std::max(kMinFramerateFps, max_frame_rate);
171 RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate;
Henrik Boströmce0ea492020-01-13 10:27:18172 source_restrictions_.set_max_frame_rate(
Henrik Boström02956fe2020-02-25 08:35:03173 max_frame_rate != std::numeric_limits<int>::max()
174 ? absl::optional<double>(max_frame_rate)
Henrik Boströmce0ea492020-01-13 10:27:18175 : absl::nullopt);
Henrik Boströmb08882b2020-01-07 09:11:17176 }
177
Henrik Boström02956fe2020-02-25 08:35:03178 bool CanIncreaseFrameRateTo(int max_frame_rate) {
179 return max_frame_rate > rtc::dchecked_cast<int>(
180 source_restrictions_.max_frame_rate().value_or(
181 std::numeric_limits<int>::max()));
182 }
183 void IncreaseFrameRateTo(int max_frame_rate) {
184 RTC_DCHECK(CanIncreaseFrameRateTo(max_frame_rate));
185 RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate;
Henrik Boströmce0ea492020-01-13 10:27:18186 source_restrictions_.set_max_frame_rate(
Henrik Boström02956fe2020-02-25 08:35:03187 max_frame_rate != std::numeric_limits<int>::max()
188 ? absl::optional<double>(max_frame_rate)
Henrik Boströmce0ea492020-01-13 10:27:18189 : absl::nullopt);
Henrik Boströmb08882b2020-01-07 09:11:17190 }
191
192 private:
Henrik Boström02956fe2020-02-25 08:35:03193 static int GetIncreasedMaxPixelsWanted(int target_pixels) {
194 if (target_pixels == std::numeric_limits<int>::max())
195 return std::numeric_limits<int>::max();
196 // When we decrease resolution, we go down to at most 3/5 of current pixels.
197 // Thus to increase resolution, we need 3/5 to get back to where we started.
198 // When going up, the desired max_pixels_per_frame() has to be significantly
199 // higher than the target because the source's native resolutions might not
200 // match the target. We pick 12/5 of the target.
201 //
202 // (This value was historically 4 times the old target, which is (3/5)*4 of
203 // the new target - or 12/5 - assuming the target is adjusted according to
204 // the above steps.)
205 RTC_DCHECK(target_pixels != std::numeric_limits<int>::max());
206 return (target_pixels * 12) / 5;
207 }
208
Henrik Boström07b17df2020-01-15 10:42:12209 VideoSourceRestrictions source_restrictions_;
Henrik Boströmb08882b2020-01-07 09:11:17210
Henrik Boströmce0ea492020-01-13 10:27:18211 RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
Henrik Boströmb08882b2020-01-07 09:11:17212};
213
Evan Shrubsolee331a122020-02-05 12:30:33214class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final {
215 public:
216 AdaptCounter() {
Henrik Boström7875c992020-02-06 09:35:00217 fps_counters_.resize(AdaptationObserverInterface::kScaleReasonSize);
218 resolution_counters_.resize(AdaptationObserverInterface::kScaleReasonSize);
219 static_assert(AdaptationObserverInterface::kScaleReasonSize == 2,
220 "Update MoveCount.");
Henrik Boströmb08882b2020-01-07 09:11:17221 }
Evan Shrubsolee331a122020-02-05 12:30:33222 ~AdaptCounter() = default;
Henrik Boströmb08882b2020-01-07 09:11:17223
Evan Shrubsolee331a122020-02-05 12:30:33224 // Get number of adaptation downscales for |reason|.
225 VideoStreamEncoderObserver::AdaptationSteps Counts(int reason) const {
226 VideoStreamEncoderObserver::AdaptationSteps counts;
227 counts.num_framerate_reductions = fps_counters_[reason];
228 counts.num_resolution_reductions = resolution_counters_[reason];
229 return counts;
Henrik Boströmb08882b2020-01-07 09:11:17230 }
Henrik Boströmb08882b2020-01-07 09:11:17231
Evan Shrubsolee331a122020-02-05 12:30:33232 std::string ToString() const {
233 rtc::StringBuilder ss;
234 ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
235 ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
236 return ss.Release();
Henrik Boströmb08882b2020-01-07 09:11:17237 }
Evan Shrubsolee331a122020-02-05 12:30:33238
239 void IncrementFramerate(int reason) { ++(fps_counters_[reason]); }
240 void IncrementResolution(int reason) { ++(resolution_counters_[reason]); }
241 void DecrementFramerate(int reason) {
242 if (fps_counters_[reason] == 0) {
243 // Balanced mode: Adapt up is in a different order, switch reason.
244 // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3).
245 // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0}
246 // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0}
247 // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0}
248 // 4. Up resolution (quality):res={quality:0,cpu:0}, fps={quality:0,cpu:0}
249 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
250 RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
251 MoveCount(&resolution_counters_, reason);
Henrik Boström7875c992020-02-06 09:35:00252 MoveCount(&fps_counters_,
253 (reason + 1) % AdaptationObserverInterface::kScaleReasonSize);
Evan Shrubsolee331a122020-02-05 12:30:33254 }
255 --(fps_counters_[reason]);
256 RTC_DCHECK_GE(fps_counters_[reason], 0);
257 }
258
259 void DecrementResolution(int reason) {
260 if (resolution_counters_[reason] == 0) {
261 // Balanced mode: Adapt up is in a different order, switch reason.
262 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
263 RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
264 MoveCount(&fps_counters_, reason);
Henrik Boström7875c992020-02-06 09:35:00265 MoveCount(&resolution_counters_,
266 (reason + 1) % AdaptationObserverInterface::kScaleReasonSize);
Evan Shrubsolee331a122020-02-05 12:30:33267 }
268 --(resolution_counters_[reason]);
269 RTC_DCHECK_GE(resolution_counters_[reason], 0);
270 }
271
272 void DecrementFramerate(int reason, int cur_fps) {
273 DecrementFramerate(reason);
274 // Reset if at max fps (i.e. in case of fewer steps up than down).
275 if (cur_fps == std::numeric_limits<int>::max())
276 absl::c_fill(fps_counters_, 0);
277 }
278
279 // Gets the total number of downgrades (for all adapt reasons).
280 int FramerateCount() const { return Count(fps_counters_); }
281 int ResolutionCount() const { return Count(resolution_counters_); }
282
283 // Gets the total number of downgrades for |reason|.
284 int FramerateCount(int reason) const { return fps_counters_[reason]; }
285 int ResolutionCount(int reason) const { return resolution_counters_[reason]; }
286 int TotalCount(int reason) const {
287 return FramerateCount(reason) + ResolutionCount(reason);
288 }
289
290 private:
291 std::string ToString(const std::vector<int>& counters) const {
292 rtc::StringBuilder ss;
Henrik Boström7875c992020-02-06 09:35:00293 for (size_t reason = 0;
294 reason < AdaptationObserverInterface::kScaleReasonSize; ++reason) {
Evan Shrubsolee331a122020-02-05 12:30:33295 ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
296 }
297 return ss.Release();
298 }
299
300 int Count(const std::vector<int>& counters) const {
301 return absl::c_accumulate(counters, 0);
302 }
303
304 void MoveCount(std::vector<int>* counters, int from_reason) {
Henrik Boström7875c992020-02-06 09:35:00305 int to_reason =
306 (from_reason + 1) % AdaptationObserverInterface::kScaleReasonSize;
Evan Shrubsolee331a122020-02-05 12:30:33307 ++((*counters)[to_reason]);
308 --((*counters)[from_reason]);
309 }
310
311 // Degradation counters holding number of framerate/resolution reductions
312 // per adapt reason.
313 std::vector<int> fps_counters_;
314 std::vector<int> resolution_counters_;
315};
Henrik Boströmb08882b2020-01-07 09:11:17316
Henrik Boström8cfecac2020-02-07 10:29:14317class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper {
318 public:
319 explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource)
320 : quality_scaler_resource_(quality_scaler_resource),
321 quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()),
322 has_seen_first_bwe_drop_(false),
323 set_start_bitrate_(DataRate::Zero()),
324 set_start_bitrate_time_ms_(0),
325 initial_framedrop_(0) {
326 RTC_DCHECK(quality_scaler_resource_);
327 }
328
329 // Output signal.
330 bool DropInitialFrames() const {
331 return initial_framedrop_ < kMaxInitialFramedrop;
332 }
333
334 // Input signals.
335 void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
336 set_start_bitrate_ = start_bitrate;
337 set_start_bitrate_time_ms_ = now_ms;
338 }
339
340 void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) {
341 if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ &&
342 quality_scaler_resource_->is_started() &&
343 quality_scaler_settings_.InitialBitrateIntervalMs() &&
344 quality_scaler_settings_.InitialBitrateFactor()) {
345 int64_t diff_ms = now_ms - set_start_bitrate_time_ms_;
346 if (diff_ms <
347 quality_scaler_settings_.InitialBitrateIntervalMs().value() &&
348 (target_bitrate <
349 (set_start_bitrate_ *
350 quality_scaler_settings_.InitialBitrateFactor().value()))) {
351 RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: "
352 << set_start_bitrate_.bps()
353 << ", target bitrate: " << target_bitrate.bps();
354 initial_framedrop_ = 0;
355 has_seen_first_bwe_drop_ = true;
356 }
357 }
358 }
359
360 void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
361
Mirko Bonadei2e161c42020-02-20 08:45:01362 void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; }
Henrik Boström8cfecac2020-02-07 10:29:14363
364 void OnQualityScalerSettingsUpdated() {
365 if (quality_scaler_resource_->is_started()) {
366 // Restart frame drops due to size.
367 initial_framedrop_ = 0;
368 } else {
369 // Quality scaling disabled so we shouldn't drop initial frames.
370 initial_framedrop_ = kMaxInitialFramedrop;
371 }
372 }
373
374 private:
375 // The maximum number of frames to drop at beginning of stream to try and
376 // achieve desired bitrate.
377 static const int kMaxInitialFramedrop = 4;
378
379 const QualityScalerResource* quality_scaler_resource_;
380 const QualityScalerSettings quality_scaler_settings_;
381 bool has_seen_first_bwe_drop_;
382 DataRate set_start_bitrate_;
383 int64_t set_start_bitrate_time_ms_;
384 // Counts how many frames we've dropped in the initial framedrop phase.
385 int initial_framedrop_;
386};
387
Henrik Boström60383832020-02-28 08:03:53388OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget::
389 AdaptationTarget(AdaptationAction action, int value)
390 : action(action), value(value) {}
391
Henrik Boströmb08882b2020-01-07 09:11:17392OveruseFrameDetectorResourceAdaptationModule::
393 OveruseFrameDetectorResourceAdaptationModule(
Evan Shrubsole7c3a1fc2020-02-04 15:26:38394 Clock* clock,
Henrik Boströmad515a22020-01-27 12:38:05395 bool experiment_cpu_load_estimator,
Henrik Boströmb08882b2020-01-07 09:11:17396 std::unique_ptr<OveruseFrameDetector> overuse_detector,
Henrik Boströmd2382002020-01-10 14:44:01397 VideoStreamEncoderObserver* encoder_stats_observer,
398 ResourceAdaptationModuleListener* adaptation_listener)
Henrik Boström07b17df2020-01-15 10:42:12399 : adaptation_listener_(adaptation_listener),
Evan Shrubsole7c3a1fc2020-02-04 15:26:38400 clock_(clock),
Evan Shrubsoleaa6fbc12020-02-25 15:26:01401 state_(State::kStopped),
Henrik Boströmad515a22020-01-27 12:38:05402 experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
Henrik Boströma3d42522020-01-16 12:55:29403 has_input_video_(false),
Henrik Boströmb08882b2020-01-07 09:11:17404 degradation_preference_(DegradationPreference::DISABLED),
405 adapt_counters_(),
406 balanced_settings_(),
407 last_adaptation_request_(absl::nullopt),
Henrik Boström8234b922020-01-13 16:26:50408 source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
Henrik Boström7875c992020-02-06 09:35:00409 encode_usage_resource_(
Henrik Boström48258ac2020-02-06 11:49:57410 std::make_unique<EncodeUsageResource>(std::move(overuse_detector))),
411 quality_scaler_resource_(std::make_unique<QualityScalerResource>()),
Henrik Boström8cfecac2020-02-07 10:29:14412 initial_frame_dropper_(std::make_unique<InitialFrameDropper>(
413 quality_scaler_resource_.get())),
Henrik Boström7875c992020-02-06 09:35:00414 quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
Henrik Boströmd4578ae2020-01-22 15:16:04415 last_input_frame_size_(absl::nullopt),
Henrik Boströmfae6f0e2020-01-20 10:16:50416 target_frame_rate_(absl::nullopt),
Evan Shrubsole7c3a1fc2020-02-04 15:26:38417 encoder_target_bitrate_bps_(absl::nullopt),
Evan Shrubsolee331a122020-02-05 12:30:33418 quality_rampup_done_(false),
419 quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()),
Henrik Boström4bab2fc2020-01-21 10:18:06420 encoder_settings_(absl::nullopt),
Henrik Boström8cfecac2020-02-07 10:29:14421 encoder_stats_observer_(encoder_stats_observer) {
Henrik Boströmd2382002020-01-10 14:44:01422 RTC_DCHECK(adaptation_listener_);
Henrik Boströmb08882b2020-01-07 09:11:17423 RTC_DCHECK(encoder_stats_observer_);
Henrik Boströmd6fb4092020-02-25 16:51:08424 ClearAdaptCounters();
Evan Shrubsoleaa6fbc12020-02-25 15:26:01425 AddResource(encode_usage_resource_.get(),
426 AdaptationObserverInterface::AdaptReason::kCpu);
427 AddResource(quality_scaler_resource_.get(),
428 AdaptationObserverInterface::AdaptReason::kQuality);
Henrik Boströmb08882b2020-01-07 09:11:17429}
430
431OveruseFrameDetectorResourceAdaptationModule::
Evan Shrubsoleaa6fbc12020-02-25 15:26:01432 ~OveruseFrameDetectorResourceAdaptationModule() {
433 RTC_DCHECK_EQ(state_, State::kStopped);
434}
Henrik Boströmb08882b2020-01-07 09:11:17435
Henrik Boströma3d42522020-01-16 12:55:29436void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation(
Henrik Boströmd2382002020-01-10 14:44:01437 ResourceAdaptationModuleListener* adaptation_listener) {
Evan Shrubsoleaa6fbc12020-02-25 15:26:01438 RTC_DCHECK_EQ(state_, State::kStopped);
Henrik Boström4bab2fc2020-01-21 10:18:06439 RTC_DCHECK(encoder_settings_.has_value());
Henrik Boström7875c992020-02-06 09:35:00440 // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener
441 // should be passed in and why. If resources are separated from modules then
442 // those resources may be started or stopped separately from the module.
Henrik Boströmd2382002020-01-10 14:44:01443 RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_);
Henrik Boström7875c992020-02-06 09:35:00444 encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions());
Evan Shrubsoleaa6fbc12020-02-25 15:26:01445 for (auto& resource_and_reason : resources_) {
446 resource_and_reason.resource->RegisterListener(this);
447 }
448 state_ = State::kStarted;
Henrik Boströmb08882b2020-01-07 09:11:17449}
450
Henrik Boströma3d42522020-01-16 12:55:29451void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() {
Henrik Boström7875c992020-02-06 09:35:00452 encode_usage_resource_->StopCheckForOveruse();
453 quality_scaler_resource_->StopCheckForOveruse();
Evan Shrubsoleaa6fbc12020-02-25 15:26:01454 for (auto& resource_and_reason : resources_) {
455 resource_and_reason.resource->UnregisterListener(this);
456 }
457 state_ = State::kStopped;
458}
459
460void OveruseFrameDetectorResourceAdaptationModule::AddResource(
461 Resource* resource) {
462 return AddResource(resource, AdaptationObserverInterface::AdaptReason::kCpu);
463}
464
465void OveruseFrameDetectorResourceAdaptationModule::AddResource(
466 Resource* resource,
467 AdaptationObserverInterface::AdaptReason reason) {
468 RTC_DCHECK(resource);
469 RTC_DCHECK(absl::c_find_if(resources_,
470 [resource](const ResourceAndReason& r) {
471 return r.resource == resource;
472 }) == resources_.end())
473 << "Resource " << resource->name() << " already was inserted";
474 resources_.emplace_back(resource, reason);
Henrik Boströmb08882b2020-01-07 09:11:17475}
476
Henrik Boströma3d42522020-01-16 12:55:29477void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo(
478 bool has_input_video) {
Henrik Boström7875c992020-02-06 09:35:00479 // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS.
Henrik Boströma3d42522020-01-16 12:55:29480 has_input_video_ = has_input_video;
481}
482
483void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference(
484 DegradationPreference degradation_preference) {
485 if (degradation_preference_ != degradation_preference) {
486 // Reset adaptation state, so that we're not tricked into thinking there's
487 // an already pending request of the same type.
488 last_adaptation_request_.reset();
489 if (degradation_preference == DegradationPreference::BALANCED ||
490 degradation_preference_ == DegradationPreference::BALANCED) {
491 // TODO(asapersson): Consider removing |adapt_counters_| map and use one
492 // AdaptCounter for all modes.
493 source_restrictor_->ClearRestrictions();
Henrik Boströmd6fb4092020-02-25 16:51:08494 ClearAdaptCounters();
Henrik Boströma3d42522020-01-16 12:55:29495 }
496 }
497 degradation_preference_ = degradation_preference;
498 MaybeUpdateVideoSourceRestrictions();
499}
500
Henrik Boström4bab2fc2020-01-21 10:18:06501void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings(
502 EncoderSettings encoder_settings) {
503 encoder_settings_ = std::move(encoder_settings);
Evan Shrubsolee331a122020-02-05 12:30:33504
505 quality_rampup_experiment_.SetMaxBitrate(
506 LastInputFrameSizeOrDefault(),
507 encoder_settings_->video_codec().maxBitrate);
Henrik Boström4bab2fc2020-01-21 10:18:06508 MaybeUpdateTargetFrameRate();
509}
510
Evan Shrubsole7c3a1fc2020-02-04 15:26:38511void OveruseFrameDetectorResourceAdaptationModule::SetStartBitrate(
512 DataRate start_bitrate) {
513 if (!start_bitrate.IsZero())
514 encoder_target_bitrate_bps_ = start_bitrate.bps();
Henrik Boström8cfecac2020-02-07 10:29:14515 initial_frame_dropper_->SetStartBitrate(start_bitrate,
516 clock_->TimeInMicroseconds());
Evan Shrubsole7c3a1fc2020-02-04 15:26:38517}
518
519void OveruseFrameDetectorResourceAdaptationModule::SetTargetBitrate(
520 DataRate target_bitrate) {
521 if (!target_bitrate.IsZero())
522 encoder_target_bitrate_bps_ = target_bitrate.bps();
Henrik Boström8cfecac2020-02-07 10:29:14523 initial_frame_dropper_->SetTargetBitrate(target_bitrate,
524 clock_->TimeInMilliseconds());
Henrik Boströmede69c02020-01-21 16:45:35525}
526
Evan Shrubsolee331a122020-02-05 12:30:33527void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates(
528 const VideoEncoder::RateControlParameters& encoder_rates) {
529 encoder_rates_ = encoder_rates;
530}
531
Henrik Boströmfae6f0e2020-01-20 10:16:50532void OveruseFrameDetectorResourceAdaptationModule::
533 ResetVideoSourceRestrictions() {
534 last_adaptation_request_.reset();
535 source_restrictor_->ClearRestrictions();
Henrik Boströmd6fb4092020-02-25 16:51:08536 ClearAdaptCounters();
Henrik Boströmfae6f0e2020-01-20 10:16:50537 MaybeUpdateVideoSourceRestrictions();
538}
539
Henrik Boströmd4578ae2020-01-22 15:16:04540void OveruseFrameDetectorResourceAdaptationModule::OnFrame(
541 const VideoFrame& frame) {
542 last_input_frame_size_ = frame.size();
Henrik Boströmb08882b2020-01-07 09:11:17543}
544
Henrik Boströmd4578ae2020-01-22 15:16:04545void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() {
Evan Shrubsole6c13fd92020-01-21 08:57:54546 int fps_count = GetConstAdaptCounter().FramerateCount(
547 AdaptationObserverInterface::AdaptReason::kQuality);
548 int res_count = GetConstAdaptCounter().ResolutionCount(
549 AdaptationObserverInterface::AdaptReason::kQuality);
Henrik Boström7875c992020-02-06 09:35:00550 OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
Evan Shrubsole6c13fd92020-01-21 08:57:54551 if (degradation_preference() == DegradationPreference::BALANCED &&
552 GetConstAdaptCounter().FramerateCount(
553 AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) {
554 // Adapt framerate in same step as resolution.
Henrik Boström7875c992020-02-06 09:35:00555 OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
Evan Shrubsole6c13fd92020-01-21 08:57:54556 }
557 if (GetConstAdaptCounter().ResolutionCount(
558 AdaptationObserverInterface::AdaptReason::kQuality) > res_count) {
559 encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
560 }
Henrik Boström8cfecac2020-02-07 10:29:14561 initial_frame_dropper_->OnFrameDroppedDueToSize();
Evan Shrubsole6c13fd92020-01-21 08:57:54562}
563
Henrik Boströmd4578ae2020-01-22 15:16:04564void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted(
565 const VideoFrame& cropped_frame,
566 int64_t time_when_first_seen_us) {
Henrik Boström7875c992020-02-06 09:35:00567 encode_usage_resource_->OnEncodeStarted(cropped_frame,
568 time_when_first_seen_us);
Henrik Boströmd4578ae2020-01-22 15:16:04569}
570
571void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted(
Evan Shrubsolebfe3ef82020-01-30 13:29:35572 const EncodedImage& encoded_image,
Henrik Boströmd4578ae2020-01-22 15:16:04573 int64_t time_sent_in_us,
Henrik Boströmd4578ae2020-01-22 15:16:04574 absl::optional<int> encode_duration_us) {
Henrik Boström7875c992020-02-06 09:35:00575 // Inform |encode_usage_resource_| of the encode completed event.
Evan Shrubsolebfe3ef82020-01-30 13:29:35576 uint32_t timestamp = encoded_image.Timestamp();
577 int64_t capture_time_us =
578 encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
Henrik Boström7875c992020-02-06 09:35:00579 encode_usage_resource_->OnEncodeCompleted(
580 timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
581 // Inform |quality_scaler_resource_| of the encode completed event.
582 quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
Henrik Boströmb08882b2020-01-07 09:11:17583}
584
Evan Shrubsolec809e8b2020-01-31 14:36:35585void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped(
586 EncodedImageCallback::DropReason reason) {
Henrik Boström7875c992020-02-06 09:35:00587 quality_scaler_resource_->OnFrameDropped(reason);
Evan Shrubsolec809e8b2020-01-31 14:36:35588}
589
Henrik Boström8cfecac2020-02-07 10:29:14590bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const {
591 return initial_frame_dropper_->DropInitialFrames();
Evan Shrubsolef2be3ef2020-02-03 09:43:31592}
593
Mirko Bonadei2e161c42020-02-20 08:45:01594void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() {
595 initial_frame_dropper_->OnMaybeEncodeFrame();
596 MaybePerformQualityRampupExperiment();
597}
598
Evan Shrubsolecf059522020-01-29 16:04:44599void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings(
600 absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
601 if (qp_thresholds.has_value()) {
Henrik Boström7875c992020-02-06 09:35:00602 quality_scaler_resource_->StopCheckForOveruse();
603 quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value());
Evan Shrubsolecf059522020-01-29 16:04:44604 } else {
Henrik Boström7875c992020-02-06 09:35:00605 quality_scaler_resource_->StopCheckForOveruse();
Evan Shrubsolecf059522020-01-29 16:04:44606 }
Henrik Boström8cfecac2020-02-07 10:29:14607 initial_frame_dropper_->OnQualityScalerSettingsUpdated();
Henrik Boströmb08882b2020-01-07 09:11:17608}
609
Evan Shrubsolec81798b2020-02-03 14:46:08610void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler(
611 const VideoEncoder::EncoderInfo& encoder_info) {
612 const auto scaling_settings = encoder_info.scaling_settings;
613 const bool quality_scaling_allowed =
614 IsResolutionScalingEnabled(degradation_preference_) &&
615 scaling_settings.thresholds;
616
Henrik Boström7875c992020-02-06 09:35:00617 // TODO(https://crbug.com/webrtc/11222): Should this move to
618 // QualityScalerResource?
Evan Shrubsolec81798b2020-02-03 14:46:08619 if (quality_scaling_allowed) {
Henrik Boström7875c992020-02-06 09:35:00620 if (!quality_scaler_resource_->is_started()) {
Evan Shrubsolec81798b2020-02-03 14:46:08621 // Quality scaler has not already been configured.
622
623 // Use experimental thresholds if available.
624 absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
625 if (quality_scaling_experiment_enabled_) {
626 experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
627 GetVideoCodecTypeOrGeneric());
628 }
629 UpdateQualityScalerSettings(experimental_thresholds
630 ? *experimental_thresholds
631 : *(scaling_settings.thresholds));
632 }
633 } else {
634 UpdateQualityScalerSettings(absl::nullopt);
635 }
636
637 // Set the qp-thresholds to the balanced settings if balanced mode.
638 if (degradation_preference_ == DegradationPreference::BALANCED &&
Henrik Boström7875c992020-02-06 09:35:00639 quality_scaler_resource_->is_started()) {
Evan Shrubsolec81798b2020-02-03 14:46:08640 absl::optional<VideoEncoder::QpThresholds> thresholds =
641 balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
642 LastInputFrameSizeOrDefault());
643 if (thresholds) {
Henrik Boström7875c992020-02-06 09:35:00644 quality_scaler_resource_->SetQpThresholds(*thresholds);
Evan Shrubsolec81798b2020-02-03 14:46:08645 }
646 }
647
648 encoder_stats_observer_->OnAdaptationChanged(
649 VideoStreamEncoderObserver::AdaptationReason::kNone,
650 GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu),
651 GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality));
652}
653
Henrik Boström48258ac2020-02-06 11:49:57654ResourceListenerResponse
655OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured(
656 const Resource& resource) {
Evan Shrubsoleaa6fbc12020-02-25 15:26:01657 const auto& registered_resource =
658 absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) {
659 return r.resource == &resource;
660 });
661 RTC_DCHECK(registered_resource != resources_.end())
662 << resource.name() << " not found.";
663
664 const AdaptationObserverInterface::AdaptReason reason =
665 registered_resource->reason;
Henrik Boström48258ac2020-02-06 11:49:57666 switch (resource.usage_state()) {
667 case ResourceUsageState::kOveruse:
668 return OnResourceOveruse(reason);
669 case ResourceUsageState::kStable:
670 // Do nothing.
671 //
672 // This module has two resources: |encoude_usage_resource_| and
673 // |quality_scaler_resource_|. A smarter adaptation module might not
674 // attempt to adapt up unless ALL resources were underused, but this
675 // module acts on each resource's measurement in isolation - without
676 // taking the current usage of any other resource into account.
677 return ResourceListenerResponse::kNothing;
678 case ResourceUsageState::kUnderuse:
679 OnResourceUnderuse(reason);
680 return ResourceListenerResponse::kNothing;
681 }
682}
683
Henrik Boström60383832020-02-28 08:03:53684absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget>
685OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget(
686 int input_pixels,
687 int input_fps,
688 AdaptationObserverInterface::AdaptReason reason) const {
689 // Preconditions for being able to adapt up:
Henrik Boströmd2a1f09b2020-02-25 08:46:36690 if (!has_input_video_)
Henrik Boström60383832020-02-28 08:03:53691 return absl::nullopt;
692 // 1. We can't adapt up if we're already at the highest setting.
Henrik Boströmd2a1f09b2020-02-25 08:46:36693 int num_downgrades = GetConstAdaptCounter().TotalCount(reason);
694 RTC_DCHECK_GE(num_downgrades, 0);
695 if (num_downgrades == 0)
Henrik Boström60383832020-02-28 08:03:53696 return absl::nullopt;
697 // 2. We shouldn't adapt up if we're currently waiting for a previous upgrade
698 // to have an effect.
Henrik Boströmd2a1f09b2020-02-25 08:46:36699 // TODO(hbos): What about in the case of other degradation preferences?
700 bool last_adaptation_was_up =
701 last_adaptation_request_ &&
702 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
703 if (last_adaptation_was_up &&
704 degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
Henrik Boström60383832020-02-28 08:03:53705 input_pixels <= last_adaptation_request_->input_pixel_count_) {
706 return absl::nullopt;
Henrik Boströmd2a1f09b2020-02-25 08:46:36707 }
Henrik Boström60383832020-02-28 08:03:53708 // 3. We shouldn't adapt up if BalancedSettings doesn't allow it, which is
709 // only applicable if reason is kQuality and preference is BALANCED.
Henrik Boströmd2a1f09b2020-02-25 08:46:36710 if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
Henrik Boströmd6fb4092020-02-25 16:51:08711 EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
Henrik Boström60383832020-02-28 08:03:53712 !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(), input_pixels,
Henrik Boströmd2a1f09b2020-02-25 08:46:36713 encoder_target_bitrate_bps_.value_or(0))) {
Henrik Boström60383832020-02-28 08:03:53714 return absl::nullopt;
Henrik Boströmd2a1f09b2020-02-25 08:46:36715 }
Henrik Boströmd2a1f09b2020-02-25 08:46:36716
Henrik Boström60383832020-02-28 08:03:53717 // Attempt to find an allowed adaptation target.
Henrik Boströmd6fb4092020-02-25 16:51:08718 switch (EffectiveDegradationPreference()) {
Henrik Boströmb08882b2020-01-07 09:11:17719 case DegradationPreference::BALANCED: {
Henrik Boström60383832020-02-28 08:03:53720 // Attempt to increase target frame rate.
721 int target_fps =
722 balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(), input_pixels);
723 if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) {
724 return AdaptationTarget(AdaptationAction::kIncreaseFrameRate,
725 target_fps);
Henrik Boströmb08882b2020-01-07 09:11:17726 }
Henrik Boström60383832020-02-28 08:03:53727 // Fall-through to maybe-adapting resolution, unless |balanced_settings_|
728 // forbids it based on bitrate.
Henrik Boström7875c992020-02-06 09:35:00729 if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
Henrik Boströmb08882b2020-01-07 09:11:17730 !balanced_settings_.CanAdaptUpResolution(
Henrik Boström60383832020-02-28 08:03:53731 GetVideoCodecTypeOrGeneric(), input_pixels,
Evan Shrubsole7c3a1fc2020-02-04 15:26:38732 encoder_target_bitrate_bps_.value_or(0))) {
Henrik Boström60383832020-02-28 08:03:53733 return absl::nullopt;
Henrik Boströmb08882b2020-01-07 09:11:17734 }
735 // Scale up resolution.
Danil Chapovalov64f1f3f2020-01-16 13:41:10736 ABSL_FALLTHROUGH_INTENDED;
Henrik Boströmb08882b2020-01-07 09:11:17737 }
738 case DegradationPreference::MAINTAIN_FRAMERATE: {
Henrik Boström60383832020-02-28 08:03:53739 // Don't adapt resolution if CanAdaptUpResolution() forbids it based on
740 // bitrate and limits specified by encoder capabilities.
Henrik Boström7875c992020-02-06 09:35:00741 if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
Henrik Boström60383832020-02-28 08:03:53742 !CanAdaptUpResolution(input_pixels,
Evan Shrubsole7c3a1fc2020-02-04 15:26:38743 encoder_target_bitrate_bps_.value_or(0))) {
Henrik Boström60383832020-02-28 08:03:53744 return absl::nullopt;
Henrik Boströmb08882b2020-01-07 09:11:17745 }
Henrik Boström60383832020-02-28 08:03:53746 // Attempt to increase pixel count.
747 int target_pixels = input_pixels;
Henrik Boströmd2a1f09b2020-02-25 08:46:36748 if (GetConstAdaptCounter().ResolutionCount() == 1) {
Henrik Boströmb08882b2020-01-07 09:11:17749 RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
Henrik Boström60383832020-02-28 08:03:53750 target_pixels = std::numeric_limits<int>::max();
Henrik Boströmb08882b2020-01-07 09:11:17751 }
Henrik Boström60383832020-02-28 08:03:53752 target_pixels =
753 VideoSourceRestrictor::GetHigherResolutionThan(target_pixels);
Henrik Boström02956fe2020-02-25 08:35:03754 if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels))
Henrik Boström60383832020-02-28 08:03:53755 return absl::nullopt;
756 return AdaptationTarget(AdaptationAction::kIncreaseResolution,
757 target_pixels);
Henrik Boströmb08882b2020-01-07 09:11:17758 }
759 case DegradationPreference::MAINTAIN_RESOLUTION: {
760 // Scale up framerate.
Henrik Boström60383832020-02-28 08:03:53761 int target_fps = input_fps;
Henrik Boströmd2a1f09b2020-02-25 08:46:36762 if (GetConstAdaptCounter().FramerateCount() == 1) {
Henrik Boströmb08882b2020-01-07 09:11:17763 RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
Henrik Boström60383832020-02-28 08:03:53764 target_fps = std::numeric_limits<int>::max();
Henrik Boströmb08882b2020-01-07 09:11:17765 }
Henrik Boström60383832020-02-28 08:03:53766 target_fps = VideoSourceRestrictor::GetHigherFrameRateThan(target_fps);
Henrik Boström02956fe2020-02-25 08:35:03767 if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps))
Henrik Boström60383832020-02-28 08:03:53768 return absl::nullopt;
769 return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
Henrik Boströmb08882b2020-01-07 09:11:17770 }
771 case DegradationPreference::DISABLED:
Henrik Boström60383832020-02-28 08:03:53772 return absl::nullopt;
Henrik Boströmb08882b2020-01-07 09:11:17773 }
Henrik Boströmd2a1f09b2020-02-25 08:46:36774}
775
Henrik Boström60383832020-02-28 08:03:53776absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget>
777OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget(
778 int input_pixels,
779 int input_fps,
780 int min_pixels_per_frame,
781 AdaptationObserverInterface::AdaptReason reason) const {
782 // Preconditions for being able to adapt down:
Henrik Boströmd2a1f09b2020-02-25 08:46:36783 if (!has_input_video_)
Henrik Boström60383832020-02-28 08:03:53784 return absl::nullopt;
785 // 1. We are not disabled.
Henrik Boströmd2a1f09b2020-02-25 08:46:36786 // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
787 // causes scaling due to bandwidth constraints (QualityScalerResource) to be
788 // ignored, not just CPU signals. This is not a use case we want to support;
789 // remove the enum value.
790 if (degradation_preference_ == DegradationPreference::DISABLED)
Henrik Boström60383832020-02-28 08:03:53791 return absl::nullopt;
Henrik Boströmd2a1f09b2020-02-25 08:46:36792 bool last_adaptation_was_down =
793 last_adaptation_request_ &&
794 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
Henrik Boström60383832020-02-28 08:03:53795 // 2. We shouldn't adapt down if our frame rate is below the minimum or if its
Henrik Boströmd2a1f09b2020-02-25 08:46:36796 // currently unknown.
Henrik Boströmd6fb4092020-02-25 16:51:08797 if (EffectiveDegradationPreference() ==
Henrik Boströmd2a1f09b2020-02-25 08:46:36798 DegradationPreference::MAINTAIN_RESOLUTION) {
799 // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
800 // - delete it.
Henrik Boström60383832020-02-28 08:03:53801 if (input_fps <= 0 ||
802 (last_adaptation_was_down && input_fps < kMinFramerateFps)) {
803 return absl::nullopt;
Henrik Boströmd2a1f09b2020-02-25 08:46:36804 }
805 }
Henrik Boström60383832020-02-28 08:03:53806 // 3. We shouldn't adapt down if we're currently waiting for a previous
807 // downgrade to have an effect.
Henrik Boströmd2a1f09b2020-02-25 08:46:36808 // TODO(hbos): What about in the case of other degradation preferences?
809 if (last_adaptation_was_down &&
810 degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
Henrik Boström60383832020-02-28 08:03:53811 input_pixels >= last_adaptation_request_->input_pixel_count_) {
812 return absl::nullopt;
Henrik Boströmd2a1f09b2020-02-25 08:46:36813 }
Henrik Boström60383832020-02-28 08:03:53814
815 // Attempt to find an allowed adaptation target.
816 switch (EffectiveDegradationPreference()) {
817 case DegradationPreference::BALANCED: {
818 // Try scale down framerate, if lower.
819 int target_fps =
820 balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(), input_pixels);
821 if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) {
822 return AdaptationTarget(AdaptationAction::kDecreaseFrameRate,
823 target_fps);
824 }
825 // Scale down resolution.
826 ABSL_FALLTHROUGH_INTENDED;
827 }
828 case DegradationPreference::MAINTAIN_FRAMERATE: {
829 // Scale down resolution.
830 int target_pixels =
831 VideoSourceRestrictor::GetLowerResolutionThan(input_pixels);
832 // TODO(https://crbug.com/webrtc/11222): Move this logic to
833 // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
834 // target is available should not have side-effects.
835 if (target_pixels < min_pixels_per_frame)
836 encoder_stats_observer_->OnMinPixelLimitReached();
837 if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels,
838 min_pixels_per_frame)) {
839 return absl::nullopt;
840 }
841 return AdaptationTarget(AdaptationAction::kDecreaseResolution,
842 target_pixels);
843 }
844 case DegradationPreference::MAINTAIN_RESOLUTION: {
845 int target_fps = VideoSourceRestrictor::GetLowerFrameRateThan(input_fps);
846 if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps))
847 return absl::nullopt;
848 return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
849 }
850 case DegradationPreference::DISABLED:
851 RTC_NOTREACHED();
852 return absl::nullopt;
853 }
854}
855
856void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget(
857 const AdaptationTarget& target,
858 int min_pixels_per_frame,
859 AdaptationObserverInterface::AdaptReason reason) {
860 switch (target.action) {
861 case AdaptationAction::kIncreaseResolution:
862 source_restrictor_->IncreaseResolutionTo(target.value);
863 GetAdaptCounter().DecrementResolution(reason);
864 return;
865 case AdaptationAction::kDecreaseResolution:
866 source_restrictor_->DecreaseResolutionTo(target.value,
867 min_pixels_per_frame);
868 GetAdaptCounter().IncrementResolution(reason);
869 return;
870 case AdaptationAction::kIncreaseFrameRate:
871 source_restrictor_->IncreaseFrameRateTo(target.value);
872 GetAdaptCounter().DecrementFramerate(reason, target.value);
873 // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
874 // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
875 // in DecrementFramerate() makes it hard to predict whether this will be
876 // the last step. Remove the dependency on GetConstAdaptCounter().
877 if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
878 GetConstAdaptCounter().FramerateCount() == 0 &&
879 target.value != std::numeric_limits<int>::max()) {
880 RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
881 source_restrictor_->IncreaseFrameRateTo(
882 std::numeric_limits<int>::max());
883 }
884 return;
885 case AdaptationAction::kDecreaseFrameRate:
886 source_restrictor_->DecreaseFrameRateTo(target.value);
887 GetAdaptCounter().IncrementFramerate(reason);
888 return;
889 }
890}
891
892void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse(
893 AdaptationObserverInterface::AdaptReason reason) {
894 int input_pixels = LastInputFrameSizeOrDefault();
895 int input_fps = encoder_stats_observer_->GetInputFrameRate();
896 int min_pixels_per_frame = MinPixelsPerFrame();
897 // Should we adapt, if so to what target?
898 absl::optional<AdaptationTarget> target =
899 GetAdaptUpTarget(input_pixels, input_fps, reason);
900 if (!target.has_value())
901 return;
902 // Apply target.
903 ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason);
904 last_adaptation_request_.emplace(AdaptationRequest{
905 input_pixels, input_fps, AdaptationRequest::Mode::kAdaptUp});
906 // Update VideoSourceRestrictions based on adaptation. This also informs the
907 // |adaptation_listener_|.
908 MaybeUpdateVideoSourceRestrictions();
909 // Stats and logging.
910 UpdateAdaptationStats(reason);
911 RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString();
Henrik Boströmb08882b2020-01-07 09:11:17912}
913
Henrik Boström48258ac2020-02-06 11:49:57914ResourceListenerResponse
915OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse(
Henrik Boström7875c992020-02-06 09:35:00916 AdaptationObserverInterface::AdaptReason reason) {
Henrik Boströma3d42522020-01-16 12:55:29917 if (!has_input_video_)
Henrik Boström48258ac2020-02-06 11:49:57918 return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency;
Henrik Boström60383832020-02-28 08:03:53919 int input_pixels = LastInputFrameSizeOrDefault();
920 int input_fps = encoder_stats_observer_->GetInputFrameRate();
921 int min_pixels_per_frame = MinPixelsPerFrame();
922 // Should we adapt, if so to what target?
923 absl::optional<AdaptationTarget> target =
924 GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame, reason);
925 if (!target.has_value())
Henrik Boströmd2a1f09b2020-02-25 08:46:36926 return ResourceListenerResponse::kNothing;
Henrik Boström60383832020-02-28 08:03:53927 // Apply target.
928 ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason);
929 last_adaptation_request_.emplace(AdaptationRequest{
930 input_pixels, input_fps, AdaptationRequest::Mode::kAdaptDown});
931 // Update VideoSourceRestrictions based on adaptation. This also informs the
932 // |adaptation_listener_|.
Henrik Boström07b17df2020-01-15 10:42:12933 MaybeUpdateVideoSourceRestrictions();
Henrik Boström60383832020-02-28 08:03:53934 // Stats and logging.
Henrik Boströmb08882b2020-01-07 09:11:17935 UpdateAdaptationStats(reason);
Henrik Boströmb08882b2020-01-07 09:11:17936 RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString();
Henrik Boström60383832020-02-28 08:03:53937 // In BALANCED, if requested FPS is higher or close to input FPS to the target
938 // we tell the QualityScaler to increase its frequency.
939 if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
940 target->action == AdaptationAction::kDecreaseFrameRate) {
941 absl::optional<int> min_diff = balanced_settings_.MinFpsDiff(input_pixels);
942 if (min_diff && input_fps > 0) {
943 int fps_diff = input_fps - target->value;
944 if (fps_diff < min_diff.value()) {
945 return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency;
946 }
947 }
948 }
949 return ResourceListenerResponse::kNothing;
Henrik Boströmb08882b2020-01-07 09:11:17950}
951
Henrik Boströmad515a22020-01-27 12:38:05952// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
953// pipelining encoders better (multiple input frames before something comes
954// out). This should effectively turn off CPU adaptations for systems that
955// remotely cope with the load right now.
956CpuOveruseOptions
957OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const {
958 // This is already ensured by the only caller of this method:
959 // StartResourceAdaptation().
960 RTC_DCHECK(encoder_settings_.has_value());
961 CpuOveruseOptions options;
962 // Hardware accelerated encoders are assumed to be pipelined; give them
963 // additional overuse time.
964 if (encoder_settings_->encoder_info().is_hardware_accelerated) {
965 options.low_encode_usage_threshold_percent = 150;
966 options.high_encode_usage_threshold_percent = 200;
967 }
968 if (experiment_cpu_load_estimator_) {
969 options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec;
970 }
971 return options;
972}
973
Henrik Boström4bab2fc2020-01-21 10:18:06974VideoCodecType
975OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric()
976 const {
977 return encoder_settings_.has_value()
978 ? encoder_settings_->encoder_config().codec_type
979 : kVideoCodecGeneric;
980}
981
Henrik Boströmd4578ae2020-01-22 15:16:04982int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault()
983 const {
984 // The dependency on this hardcoded resolution is inherited from old code,
985 // which used this resolution as a stand-in for not knowing the resolution
986 // yet.
987 // TODO(hbos): Can we simply DCHECK has_value() before usage instead? Having a
988 // DCHECK passed all the tests but adding it does change the requirements of
Henrik Boström7875c992020-02-06 09:35:00989 // this class (= not being allowed to call OnResourceUnderuse() or
990 // OnResourceOveruse() before OnFrame()) and deserves a standalone CL.
Henrik Boströmd4578ae2020-01-22 15:16:04991 return last_input_frame_size_.value_or(
992 VideoStreamEncoder::kDefaultLastFrameInfoWidth *
993 VideoStreamEncoder::kDefaultLastFrameInfoHeight);
994}
995
Henrik Boström60383832020-02-28 08:03:53996int OveruseFrameDetectorResourceAdaptationModule::MinPixelsPerFrame() const {
997 return encoder_settings_.has_value()
998 ? encoder_settings_->encoder_info()
999 .scaling_settings.min_pixels_per_frame
1000 : kDefaultMinPixelsPerFrame;
1001}
1002
Henrik Boström8234b922020-01-13 16:26:501003void OveruseFrameDetectorResourceAdaptationModule::
Henrik Boström07b17df2020-01-15 10:42:121004 MaybeUpdateVideoSourceRestrictions() {
1005 VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
1006 source_restrictor_->source_restrictions(), degradation_preference_);
1007 if (video_source_restrictions_ != new_restrictions) {
1008 video_source_restrictions_ = std::move(new_restrictions);
Henrik Boström8234b922020-01-13 16:26:501009 adaptation_listener_->OnVideoSourceRestrictionsUpdated(
Henrik Boström07b17df2020-01-15 10:42:121010 video_source_restrictions_);
Henrik Boströmfae6f0e2020-01-20 10:16:501011 MaybeUpdateTargetFrameRate();
1012 }
1013}
1014
1015void OveruseFrameDetectorResourceAdaptationModule::
1016 MaybeUpdateTargetFrameRate() {
Henrik Boström4bab2fc2020-01-21 10:18:061017 absl::optional<double> codec_max_frame_rate =
1018 encoder_settings_.has_value()
1019 ? absl::optional<double>(
1020 encoder_settings_->video_codec().maxFramerate)
1021 : absl::nullopt;
Henrik Boströmfae6f0e2020-01-20 10:16:501022 // The current target framerate is the maximum frame rate as specified by
1023 // the current codec configuration or any limit imposed by the adaptation
1024 // module. This is used to make sure overuse detection doesn't needlessly
1025 // trigger in low and/or variable framerate scenarios.
1026 absl::optional<double> target_frame_rate =
1027 ApplyDegradationPreference(source_restrictor_->source_restrictions(),
1028 degradation_preference_)
1029 .max_frame_rate();
1030 if (!target_frame_rate.has_value() ||
Henrik Boström4bab2fc2020-01-21 10:18:061031 (codec_max_frame_rate.has_value() &&
1032 codec_max_frame_rate.value() < target_frame_rate.value())) {
1033 target_frame_rate = codec_max_frame_rate;
Henrik Boströmfae6f0e2020-01-20 10:16:501034 }
Henrik Boström7875c992020-02-06 09:35:001035 encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
Henrik Boström8234b922020-01-13 16:26:501036}
1037
Henrik Boströmb08882b2020-01-07 09:11:171038// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
1039void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats(
Henrik Boström7875c992020-02-06 09:35:001040 AdaptationObserverInterface::AdaptReason reason) {
Henrik Boströmb08882b2020-01-07 09:11:171041 switch (reason) {
Henrik Boström7875c992020-02-06 09:35:001042 case AdaptationObserverInterface::AdaptReason::kCpu:
Henrik Boströmb08882b2020-01-07 09:11:171043 encoder_stats_observer_->OnAdaptationChanged(
1044 VideoStreamEncoderObserver::AdaptationReason::kCpu,
Henrik Boström7875c992020-02-06 09:35:001045 GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu),
1046 GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality));
Henrik Boströmb08882b2020-01-07 09:11:171047 break;
Henrik Boström7875c992020-02-06 09:35:001048 case AdaptationObserverInterface::AdaptReason::kQuality:
Henrik Boströmb08882b2020-01-07 09:11:171049 encoder_stats_observer_->OnAdaptationChanged(
1050 VideoStreamEncoderObserver::AdaptationReason::kQuality,
Henrik Boström7875c992020-02-06 09:35:001051 GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu),
1052 GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality));
Henrik Boströmb08882b2020-01-07 09:11:171053 break;
1054 }
1055}
1056
1057VideoStreamEncoderObserver::AdaptationSteps
1058OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
Henrik Boström7875c992020-02-06 09:35:001059 AdaptationObserverInterface::AdaptReason reason) {
Henrik Boströmb08882b2020-01-07 09:11:171060 VideoStreamEncoderObserver::AdaptationSteps counts =
1061 GetConstAdaptCounter().Counts(reason);
1062 switch (reason) {
Henrik Boström7875c992020-02-06 09:35:001063 case AdaptationObserverInterface::AdaptReason::kCpu:
Henrik Boströmb08882b2020-01-07 09:11:171064 if (!IsFramerateScalingEnabled(degradation_preference_))
1065 counts.num_framerate_reductions = absl::nullopt;
1066 if (!IsResolutionScalingEnabled(degradation_preference_))
1067 counts.num_resolution_reductions = absl::nullopt;
1068 break;
Henrik Boström7875c992020-02-06 09:35:001069 case AdaptationObserverInterface::AdaptReason::kQuality:
Henrik Boströmb08882b2020-01-07 09:11:171070 if (!IsFramerateScalingEnabled(degradation_preference_) ||
Henrik Boström7875c992020-02-06 09:35:001071 !quality_scaler_resource_->is_started()) {
Henrik Boströmb08882b2020-01-07 09:11:171072 counts.num_framerate_reductions = absl::nullopt;
1073 }
1074 if (!IsResolutionScalingEnabled(degradation_preference_) ||
Henrik Boström7875c992020-02-06 09:35:001075 !quality_scaler_resource_->is_started()) {
Henrik Boströmb08882b2020-01-07 09:11:171076 counts.num_resolution_reductions = absl::nullopt;
1077 }
1078 break;
1079 }
1080 return counts;
1081}
1082
Henrik Boströmd6fb4092020-02-25 16:51:081083DegradationPreference
1084OveruseFrameDetectorResourceAdaptationModule::EffectiveDegradationPreference()
1085 const {
Henrik Boströmb08882b2020-01-07 09:11:171086 // Balanced mode for screenshare works via automatic animation detection:
1087 // Resolution is capped for fullscreen animated content.
1088 // Adapatation is done only via framerate downgrade.
1089 // Thus effective degradation preference is MAINTAIN_RESOLUTION.
Henrik Boström4bab2fc2020-01-21 10:18:061090 return (encoder_settings_.has_value() &&
1091 encoder_settings_->encoder_config().content_type ==
Henrik Boströmb08882b2020-01-07 09:11:171092 VideoEncoderConfig::ContentType::kScreen &&
1093 degradation_preference_ == DegradationPreference::BALANCED)
1094 ? DegradationPreference::MAINTAIN_RESOLUTION
1095 : degradation_preference_;
1096}
1097
1098OveruseFrameDetectorResourceAdaptationModule::AdaptCounter&
1099OveruseFrameDetectorResourceAdaptationModule::GetAdaptCounter() {
1100 return adapt_counters_[degradation_preference_];
1101}
1102
Henrik Boströmd6fb4092020-02-25 16:51:081103void OveruseFrameDetectorResourceAdaptationModule::ClearAdaptCounters() {
1104 adapt_counters_.clear();
1105 adapt_counters_.insert(
1106 std::make_pair(DegradationPreference::DISABLED, AdaptCounter()));
1107 adapt_counters_.insert(std::make_pair(
1108 DegradationPreference::MAINTAIN_FRAMERATE, AdaptCounter()));
1109 adapt_counters_.insert(std::make_pair(
1110 DegradationPreference::MAINTAIN_RESOLUTION, AdaptCounter()));
1111 adapt_counters_.insert(
1112 std::make_pair(DegradationPreference::BALANCED, AdaptCounter()));
1113}
1114
Henrik Boströmb08882b2020-01-07 09:11:171115const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter&
Henrik Boströmd6fb4092020-02-25 16:51:081116OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() const {
1117 auto it = adapt_counters_.find(degradation_preference_);
1118 RTC_DCHECK(it != adapt_counters_.cend());
1119 return it->second;
Henrik Boströmb08882b2020-01-07 09:11:171120}
1121
Henrik Boströmb08882b2020-01-07 09:11:171122bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
1123 int pixels,
1124 uint32_t bitrate_bps) const {
1125 absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
Henrik Boström4bab2fc2020-01-21 10:18:061126 encoder_settings_.has_value()
1127 ? GetEncoderBitrateLimits(
1128 encoder_settings_->encoder_info(),
Henrik Boström02956fe2020-02-25 08:35:031129 VideoSourceRestrictor::GetHigherResolutionThan(pixels))
Henrik Boström4bab2fc2020-01-21 10:18:061130 : absl::nullopt;
Henrik Boströmb08882b2020-01-07 09:11:171131 if (!bitrate_limits.has_value() || bitrate_bps == 0) {
1132 return true; // No limit configured or bitrate provided.
1133 }
1134 RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels);
1135 return bitrate_bps >=
1136 static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
1137}
Evan Shrubsolee331a122020-02-05 12:30:331138void OveruseFrameDetectorResourceAdaptationModule::
1139 MaybePerformQualityRampupExperiment() {
Henrik Boström7875c992020-02-06 09:35:001140 if (!quality_scaler_resource_->is_started())
Evan Shrubsolee331a122020-02-05 12:30:331141 return;
1142
1143 if (quality_rampup_done_)
1144 return;
1145
1146 int64_t now_ms = clock_->TimeInMilliseconds();
1147 uint32_t bw_kbps = encoder_rates_.has_value()
1148 ? encoder_rates_.value().bandwidth_allocation.kbps()
1149 : 0;
1150
1151 bool try_quality_rampup = false;
1152 if (quality_rampup_experiment_.BwHigh(now_ms, bw_kbps)) {
1153 // Verify that encoder is at max bitrate and the QP is low.
1154 if (encoder_settings_ &&
1155 encoder_target_bitrate_bps_.value_or(0) ==
1156 encoder_settings_->video_codec().maxBitrate * 1000 &&
Henrik Boström7875c992020-02-06 09:35:001157 quality_scaler_resource_->QpFastFilterLow()) {
Evan Shrubsolee331a122020-02-05 12:30:331158 try_quality_rampup = true;
1159 }
1160 }
1161 if (try_quality_rampup &&
1162 GetConstAdaptCounter().ResolutionCount(
1163 AdaptationObserverInterface::AdaptReason::kQuality) > 0 &&
1164 GetConstAdaptCounter().TotalCount(
1165 AdaptationObserverInterface::AdaptReason::kCpu) == 0) {
1166 RTC_LOG(LS_INFO) << "Reset quality limitations.";
1167 ResetVideoSourceRestrictions();
1168 quality_rampup_done_ = true;
1169 }
1170}
1171
Henrik Boströmb08882b2020-01-07 09:11:171172} // namespace webrtc