Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2019 The WebRTC Project Authors. All rights reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include "video/overuse_frame_detector_resource_adaptation_module.h" |
| 12 | |
| 13 | #include <algorithm> |
| 14 | #include <limits> |
| 15 | #include <memory> |
| 16 | #include <string> |
| 17 | #include <utility> |
| 18 | |
| 19 | #include "absl/algorithm/container.h" |
Danil Chapovalov | 64f1f3f | 2020-01-16 13:41:10 | [diff] [blame] | 20 | #include "absl/base/macros.h" |
Henrik Boström | 07b17df | 2020-01-15 10:42:12 | [diff] [blame] | 21 | #include "api/task_queue/task_queue_base.h" |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 22 | #include "api/video/video_source_interface.h" |
Henrik Boström | 48258ac | 2020-02-06 11:49:57 | [diff] [blame] | 23 | #include "call/adaptation/resource.h" |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 24 | #include "call/adaptation/video_source_restrictions.h" |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 25 | #include "rtc_base/logging.h" |
Henrik Boström | d238200 | 2020-01-10 14:44:01 | [diff] [blame] | 26 | #include "rtc_base/numerics/safe_conversions.h" |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 27 | #include "rtc_base/strings/string_builder.h" |
Henrik Boström | ad515a2 | 2020-01-27 12:38:05 | [diff] [blame] | 28 | #include "rtc_base/time_utils.h" |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 29 | #include "video/video_stream_encoder.h" |
| 30 | |
| 31 | namespace webrtc { |
| 32 | |
| 33 | namespace { |
| 34 | |
| 35 | const int kMinFramerateFps = 2; |
| 36 | |
| 37 | bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { |
| 38 | return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE || |
| 39 | degradation_preference == DegradationPreference::BALANCED; |
| 40 | } |
| 41 | |
| 42 | bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { |
| 43 | return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION || |
| 44 | degradation_preference == DegradationPreference::BALANCED; |
| 45 | } |
| 46 | |
Henrik Boström | 8234b92 | 2020-01-13 16:26:50 | [diff] [blame] | 47 | // Returns modified restrictions where any constraints that don't apply to the |
| 48 | // degradation preference are cleared. |
| 49 | VideoSourceRestrictions ApplyDegradationPreference( |
| 50 | VideoSourceRestrictions source_restrictions, |
| 51 | DegradationPreference degradation_preference) { |
| 52 | switch (degradation_preference) { |
| 53 | case DegradationPreference::BALANCED: |
| 54 | break; |
| 55 | case DegradationPreference::MAINTAIN_FRAMERATE: |
| 56 | source_restrictions.set_max_frame_rate(absl::nullopt); |
| 57 | break; |
| 58 | case DegradationPreference::MAINTAIN_RESOLUTION: |
| 59 | source_restrictions.set_max_pixels_per_frame(absl::nullopt); |
| 60 | source_restrictions.set_target_pixels_per_frame(absl::nullopt); |
| 61 | break; |
| 62 | case DegradationPreference::DISABLED: |
| 63 | source_restrictions.set_max_pixels_per_frame(absl::nullopt); |
| 64 | source_restrictions.set_target_pixels_per_frame(absl::nullopt); |
| 65 | source_restrictions.set_max_frame_rate(absl::nullopt); |
| 66 | } |
| 67 | return source_restrictions; |
| 68 | } |
| 69 | |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 70 | } // namespace |
| 71 | |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 72 | // VideoSourceRestrictor is responsible for keeping track of current |
| 73 | // VideoSourceRestrictions and how to modify them in response to adapting up or |
| 74 | // down. It is not reponsible for determining when we should adapt up or down - |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 75 | // for that, see |
| 76 | // OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse() and |
| 77 | // OnResourceOveruse() - only how to modify the source/sink restrictions when |
| 78 | // this happens. Note that it is also not responsible for reconfigruring the |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 79 | // source/sink, it is only a keeper of desired restrictions. |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 80 | class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 81 | public: |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 82 | // For frame rate, the steps we take are 2/3 (down) and 3/2 (up). |
| 83 | static int GetLowerFrameRateThan(int fps) { |
| 84 | RTC_DCHECK(fps != std::numeric_limits<int>::max()); |
| 85 | return (fps * 2) / 3; |
| 86 | } |
| 87 | // TODO(hbos): Use absl::optional<> instead? |
| 88 | static int GetHigherFrameRateThan(int fps) { |
| 89 | return fps != std::numeric_limits<int>::max() |
| 90 | ? (fps * 3) / 2 |
| 91 | : std::numeric_limits<int>::max(); |
| 92 | } |
| 93 | |
| 94 | // For resolution, the steps we take are 3/5 (down) and 5/3 (up). |
| 95 | // Notice the asymmetry of which restriction property is set depending on if |
| 96 | // we are adapting up or down: |
| 97 | // - DecreaseResolution() sets the max_pixels_per_frame() to the desired |
| 98 | // target and target_pixels_per_frame() to null. |
| 99 | // - IncreaseResolutionTo() sets the target_pixels_per_frame() to the desired |
| 100 | // target, and max_pixels_per_frame() is set according to |
| 101 | // GetIncreasedMaxPixelsWanted(). |
| 102 | static int GetLowerResolutionThan(int pixel_count) { |
| 103 | RTC_DCHECK(pixel_count != std::numeric_limits<int>::max()); |
| 104 | return (pixel_count * 3) / 5; |
| 105 | } |
| 106 | // TODO(hbos): Use absl::optional<> instead? |
| 107 | static int GetHigherResolutionThan(int pixel_count) { |
| 108 | return pixel_count != std::numeric_limits<int>::max() |
| 109 | ? (pixel_count * 5) / 3 |
| 110 | : std::numeric_limits<int>::max(); |
| 111 | } |
| 112 | |
Henrik Boström | a3d4252 | 2020-01-16 12:55:29 | [diff] [blame] | 113 | VideoSourceRestrictor() {} |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 114 | |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 115 | VideoSourceRestrictions source_restrictions() { |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 116 | return source_restrictions_; |
Henrik Boström | d238200 | 2020-01-10 14:44:01 | [diff] [blame] | 117 | } |
Henrik Boström | 8234b92 | 2020-01-13 16:26:50 | [diff] [blame] | 118 | void ClearRestrictions() { |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 119 | source_restrictions_ = VideoSourceRestrictions(); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 120 | } |
| 121 | |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 122 | bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) { |
| 123 | int max_pixels_per_frame = rtc::dchecked_cast<int>( |
| 124 | source_restrictions_.max_pixels_per_frame().value_or( |
| 125 | std::numeric_limits<int>::max())); |
| 126 | return target_pixels < max_pixels_per_frame && |
| 127 | target_pixels >= min_pixels_per_frame; |
| 128 | } |
| 129 | void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) { |
| 130 | RTC_DCHECK(CanDecreaseResolutionTo(target_pixels, min_pixels_per_frame)); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 131 | RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 132 | << target_pixels; |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 133 | source_restrictions_.set_max_pixels_per_frame( |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 134 | target_pixels != std::numeric_limits<int>::max() |
| 135 | ? absl::optional<size_t>(target_pixels) |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 136 | : absl::nullopt); |
| 137 | source_restrictions_.set_target_pixels_per_frame(absl::nullopt); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 138 | } |
| 139 | |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 140 | bool CanIncreaseResolutionTo(int target_pixels) { |
| 141 | int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); |
| 142 | int max_pixels_per_frame = rtc::dchecked_cast<int>( |
| 143 | source_restrictions_.max_pixels_per_frame().value_or( |
| 144 | std::numeric_limits<int>::max())); |
| 145 | return max_pixels_wanted > max_pixels_per_frame; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 146 | } |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 147 | void IncreaseResolutionTo(int target_pixels) { |
| 148 | RTC_DCHECK(CanIncreaseResolutionTo(target_pixels)); |
| 149 | int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 150 | RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " |
| 151 | << max_pixels_wanted; |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 152 | source_restrictions_.set_max_pixels_per_frame( |
| 153 | max_pixels_wanted != std::numeric_limits<int>::max() |
| 154 | ? absl::optional<size_t>(max_pixels_wanted) |
| 155 | : absl::nullopt); |
| 156 | source_restrictions_.set_target_pixels_per_frame( |
| 157 | max_pixels_wanted != std::numeric_limits<int>::max() |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 158 | ? absl::optional<size_t>(target_pixels) |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 159 | : absl::nullopt); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 160 | } |
| 161 | |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 162 | bool CanDecreaseFrameRateTo(int max_frame_rate) { |
| 163 | const int fps_wanted = std::max(kMinFramerateFps, max_frame_rate); |
| 164 | return fps_wanted < rtc::dchecked_cast<int>( |
| 165 | source_restrictions_.max_frame_rate().value_or( |
| 166 | std::numeric_limits<int>::max())); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 167 | } |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 168 | void DecreaseFrameRateTo(int max_frame_rate) { |
| 169 | RTC_DCHECK(CanDecreaseFrameRateTo(max_frame_rate)); |
| 170 | max_frame_rate = std::max(kMinFramerateFps, max_frame_rate); |
| 171 | RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate; |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 172 | source_restrictions_.set_max_frame_rate( |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 173 | max_frame_rate != std::numeric_limits<int>::max() |
| 174 | ? absl::optional<double>(max_frame_rate) |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 175 | : absl::nullopt); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 176 | } |
| 177 | |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 178 | bool CanIncreaseFrameRateTo(int max_frame_rate) { |
| 179 | return max_frame_rate > rtc::dchecked_cast<int>( |
| 180 | source_restrictions_.max_frame_rate().value_or( |
| 181 | std::numeric_limits<int>::max())); |
| 182 | } |
| 183 | void IncreaseFrameRateTo(int max_frame_rate) { |
| 184 | RTC_DCHECK(CanIncreaseFrameRateTo(max_frame_rate)); |
| 185 | RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate; |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 186 | source_restrictions_.set_max_frame_rate( |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 187 | max_frame_rate != std::numeric_limits<int>::max() |
| 188 | ? absl::optional<double>(max_frame_rate) |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 189 | : absl::nullopt); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 190 | } |
| 191 | |
| 192 | private: |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 193 | static int GetIncreasedMaxPixelsWanted(int target_pixels) { |
| 194 | if (target_pixels == std::numeric_limits<int>::max()) |
| 195 | return std::numeric_limits<int>::max(); |
| 196 | // When we decrease resolution, we go down to at most 3/5 of current pixels. |
| 197 | // Thus to increase resolution, we need 3/5 to get back to where we started. |
| 198 | // When going up, the desired max_pixels_per_frame() has to be significantly |
| 199 | // higher than the target because the source's native resolutions might not |
| 200 | // match the target. We pick 12/5 of the target. |
| 201 | // |
| 202 | // (This value was historically 4 times the old target, which is (3/5)*4 of |
| 203 | // the new target - or 12/5 - assuming the target is adjusted according to |
| 204 | // the above steps.) |
| 205 | RTC_DCHECK(target_pixels != std::numeric_limits<int>::max()); |
| 206 | return (target_pixels * 12) / 5; |
| 207 | } |
| 208 | |
Henrik Boström | 07b17df | 2020-01-15 10:42:12 | [diff] [blame] | 209 | VideoSourceRestrictions source_restrictions_; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 210 | |
Henrik Boström | ce0ea49 | 2020-01-13 10:27:18 | [diff] [blame] | 211 | RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 212 | }; |
| 213 | |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 214 | class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { |
| 215 | public: |
| 216 | AdaptCounter() { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 217 | fps_counters_.resize(AdaptationObserverInterface::kScaleReasonSize); |
| 218 | resolution_counters_.resize(AdaptationObserverInterface::kScaleReasonSize); |
| 219 | static_assert(AdaptationObserverInterface::kScaleReasonSize == 2, |
| 220 | "Update MoveCount."); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 221 | } |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 222 | ~AdaptCounter() = default; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 223 | |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 224 | // Get number of adaptation downscales for |reason|. |
| 225 | VideoStreamEncoderObserver::AdaptationSteps Counts(int reason) const { |
| 226 | VideoStreamEncoderObserver::AdaptationSteps counts; |
| 227 | counts.num_framerate_reductions = fps_counters_[reason]; |
| 228 | counts.num_resolution_reductions = resolution_counters_[reason]; |
| 229 | return counts; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 230 | } |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 231 | |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 232 | std::string ToString() const { |
| 233 | rtc::StringBuilder ss; |
| 234 | ss << "Downgrade counts: fps: {" << ToString(fps_counters_); |
| 235 | ss << "}, resolution: {" << ToString(resolution_counters_) << "}"; |
| 236 | return ss.Release(); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 237 | } |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 238 | |
| 239 | void IncrementFramerate(int reason) { ++(fps_counters_[reason]); } |
| 240 | void IncrementResolution(int reason) { ++(resolution_counters_[reason]); } |
| 241 | void DecrementFramerate(int reason) { |
| 242 | if (fps_counters_[reason] == 0) { |
| 243 | // Balanced mode: Adapt up is in a different order, switch reason. |
| 244 | // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3). |
| 245 | // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0} |
| 246 | // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0} |
| 247 | // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0} |
| 248 | // 4. Up resolution (quality):res={quality:0,cpu:0}, fps={quality:0,cpu:0} |
| 249 | RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; |
| 250 | RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; |
| 251 | MoveCount(&resolution_counters_, reason); |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 252 | MoveCount(&fps_counters_, |
| 253 | (reason + 1) % AdaptationObserverInterface::kScaleReasonSize); |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 254 | } |
| 255 | --(fps_counters_[reason]); |
| 256 | RTC_DCHECK_GE(fps_counters_[reason], 0); |
| 257 | } |
| 258 | |
| 259 | void DecrementResolution(int reason) { |
| 260 | if (resolution_counters_[reason] == 0) { |
| 261 | // Balanced mode: Adapt up is in a different order, switch reason. |
| 262 | RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; |
| 263 | RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; |
| 264 | MoveCount(&fps_counters_, reason); |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 265 | MoveCount(&resolution_counters_, |
| 266 | (reason + 1) % AdaptationObserverInterface::kScaleReasonSize); |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 267 | } |
| 268 | --(resolution_counters_[reason]); |
| 269 | RTC_DCHECK_GE(resolution_counters_[reason], 0); |
| 270 | } |
| 271 | |
| 272 | void DecrementFramerate(int reason, int cur_fps) { |
| 273 | DecrementFramerate(reason); |
| 274 | // Reset if at max fps (i.e. in case of fewer steps up than down). |
| 275 | if (cur_fps == std::numeric_limits<int>::max()) |
| 276 | absl::c_fill(fps_counters_, 0); |
| 277 | } |
| 278 | |
| 279 | // Gets the total number of downgrades (for all adapt reasons). |
| 280 | int FramerateCount() const { return Count(fps_counters_); } |
| 281 | int ResolutionCount() const { return Count(resolution_counters_); } |
| 282 | |
| 283 | // Gets the total number of downgrades for |reason|. |
| 284 | int FramerateCount(int reason) const { return fps_counters_[reason]; } |
| 285 | int ResolutionCount(int reason) const { return resolution_counters_[reason]; } |
| 286 | int TotalCount(int reason) const { |
| 287 | return FramerateCount(reason) + ResolutionCount(reason); |
| 288 | } |
| 289 | |
| 290 | private: |
| 291 | std::string ToString(const std::vector<int>& counters) const { |
| 292 | rtc::StringBuilder ss; |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 293 | for (size_t reason = 0; |
| 294 | reason < AdaptationObserverInterface::kScaleReasonSize; ++reason) { |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 295 | ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; |
| 296 | } |
| 297 | return ss.Release(); |
| 298 | } |
| 299 | |
| 300 | int Count(const std::vector<int>& counters) const { |
| 301 | return absl::c_accumulate(counters, 0); |
| 302 | } |
| 303 | |
| 304 | void MoveCount(std::vector<int>* counters, int from_reason) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 305 | int to_reason = |
| 306 | (from_reason + 1) % AdaptationObserverInterface::kScaleReasonSize; |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 307 | ++((*counters)[to_reason]); |
| 308 | --((*counters)[from_reason]); |
| 309 | } |
| 310 | |
| 311 | // Degradation counters holding number of framerate/resolution reductions |
| 312 | // per adapt reason. |
| 313 | std::vector<int> fps_counters_; |
| 314 | std::vector<int> resolution_counters_; |
| 315 | }; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 316 | |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 317 | class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper { |
| 318 | public: |
| 319 | explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource) |
| 320 | : quality_scaler_resource_(quality_scaler_resource), |
| 321 | quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), |
| 322 | has_seen_first_bwe_drop_(false), |
| 323 | set_start_bitrate_(DataRate::Zero()), |
| 324 | set_start_bitrate_time_ms_(0), |
| 325 | initial_framedrop_(0) { |
| 326 | RTC_DCHECK(quality_scaler_resource_); |
| 327 | } |
| 328 | |
| 329 | // Output signal. |
| 330 | bool DropInitialFrames() const { |
| 331 | return initial_framedrop_ < kMaxInitialFramedrop; |
| 332 | } |
| 333 | |
| 334 | // Input signals. |
| 335 | void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) { |
| 336 | set_start_bitrate_ = start_bitrate; |
| 337 | set_start_bitrate_time_ms_ = now_ms; |
| 338 | } |
| 339 | |
| 340 | void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) { |
| 341 | if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ && |
| 342 | quality_scaler_resource_->is_started() && |
| 343 | quality_scaler_settings_.InitialBitrateIntervalMs() && |
| 344 | quality_scaler_settings_.InitialBitrateFactor()) { |
| 345 | int64_t diff_ms = now_ms - set_start_bitrate_time_ms_; |
| 346 | if (diff_ms < |
| 347 | quality_scaler_settings_.InitialBitrateIntervalMs().value() && |
| 348 | (target_bitrate < |
| 349 | (set_start_bitrate_ * |
| 350 | quality_scaler_settings_.InitialBitrateFactor().value()))) { |
| 351 | RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: " |
| 352 | << set_start_bitrate_.bps() |
| 353 | << ", target bitrate: " << target_bitrate.bps(); |
| 354 | initial_framedrop_ = 0; |
| 355 | has_seen_first_bwe_drop_ = true; |
| 356 | } |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | void OnFrameDroppedDueToSize() { ++initial_framedrop_; } |
| 361 | |
Mirko Bonadei | 2e161c4 | 2020-02-20 08:45:01 | [diff] [blame] | 362 | void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; } |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 363 | |
| 364 | void OnQualityScalerSettingsUpdated() { |
| 365 | if (quality_scaler_resource_->is_started()) { |
| 366 | // Restart frame drops due to size. |
| 367 | initial_framedrop_ = 0; |
| 368 | } else { |
| 369 | // Quality scaling disabled so we shouldn't drop initial frames. |
| 370 | initial_framedrop_ = kMaxInitialFramedrop; |
| 371 | } |
| 372 | } |
| 373 | |
| 374 | private: |
| 375 | // The maximum number of frames to drop at beginning of stream to try and |
| 376 | // achieve desired bitrate. |
| 377 | static const int kMaxInitialFramedrop = 4; |
| 378 | |
| 379 | const QualityScalerResource* quality_scaler_resource_; |
| 380 | const QualityScalerSettings quality_scaler_settings_; |
| 381 | bool has_seen_first_bwe_drop_; |
| 382 | DataRate set_start_bitrate_; |
| 383 | int64_t set_start_bitrate_time_ms_; |
| 384 | // Counts how many frames we've dropped in the initial framedrop phase. |
| 385 | int initial_framedrop_; |
| 386 | }; |
| 387 | |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 388 | OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget:: |
| 389 | AdaptationTarget(AdaptationAction action, int value) |
| 390 | : action(action), value(value) {} |
| 391 | |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 392 | OveruseFrameDetectorResourceAdaptationModule:: |
| 393 | OveruseFrameDetectorResourceAdaptationModule( |
Evan Shrubsole | 7c3a1fc | 2020-02-04 15:26:38 | [diff] [blame] | 394 | Clock* clock, |
Henrik Boström | ad515a2 | 2020-01-27 12:38:05 | [diff] [blame] | 395 | bool experiment_cpu_load_estimator, |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 396 | std::unique_ptr<OveruseFrameDetector> overuse_detector, |
Henrik Boström | d238200 | 2020-01-10 14:44:01 | [diff] [blame] | 397 | VideoStreamEncoderObserver* encoder_stats_observer, |
| 398 | ResourceAdaptationModuleListener* adaptation_listener) |
Henrik Boström | 07b17df | 2020-01-15 10:42:12 | [diff] [blame] | 399 | : adaptation_listener_(adaptation_listener), |
Evan Shrubsole | 7c3a1fc | 2020-02-04 15:26:38 | [diff] [blame] | 400 | clock_(clock), |
Evan Shrubsole | aa6fbc1 | 2020-02-25 15:26:01 | [diff] [blame] | 401 | state_(State::kStopped), |
Henrik Boström | ad515a2 | 2020-01-27 12:38:05 | [diff] [blame] | 402 | experiment_cpu_load_estimator_(experiment_cpu_load_estimator), |
Henrik Boström | a3d4252 | 2020-01-16 12:55:29 | [diff] [blame] | 403 | has_input_video_(false), |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 404 | degradation_preference_(DegradationPreference::DISABLED), |
| 405 | adapt_counters_(), |
| 406 | balanced_settings_(), |
| 407 | last_adaptation_request_(absl::nullopt), |
Henrik Boström | 8234b92 | 2020-01-13 16:26:50 | [diff] [blame] | 408 | source_restrictor_(std::make_unique<VideoSourceRestrictor>()), |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 409 | encode_usage_resource_( |
Henrik Boström | 48258ac | 2020-02-06 11:49:57 | [diff] [blame] | 410 | std::make_unique<EncodeUsageResource>(std::move(overuse_detector))), |
| 411 | quality_scaler_resource_(std::make_unique<QualityScalerResource>()), |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 412 | initial_frame_dropper_(std::make_unique<InitialFrameDropper>( |
| 413 | quality_scaler_resource_.get())), |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 414 | quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 415 | last_input_frame_size_(absl::nullopt), |
Henrik Boström | fae6f0e | 2020-01-20 10:16:50 | [diff] [blame] | 416 | target_frame_rate_(absl::nullopt), |
Evan Shrubsole | 7c3a1fc | 2020-02-04 15:26:38 | [diff] [blame] | 417 | encoder_target_bitrate_bps_(absl::nullopt), |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 418 | quality_rampup_done_(false), |
| 419 | quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 420 | encoder_settings_(absl::nullopt), |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 421 | encoder_stats_observer_(encoder_stats_observer) { |
Henrik Boström | d238200 | 2020-01-10 14:44:01 | [diff] [blame] | 422 | RTC_DCHECK(adaptation_listener_); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 423 | RTC_DCHECK(encoder_stats_observer_); |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 424 | ClearAdaptCounters(); |
Evan Shrubsole | aa6fbc1 | 2020-02-25 15:26:01 | [diff] [blame] | 425 | AddResource(encode_usage_resource_.get(), |
| 426 | AdaptationObserverInterface::AdaptReason::kCpu); |
| 427 | AddResource(quality_scaler_resource_.get(), |
| 428 | AdaptationObserverInterface::AdaptReason::kQuality); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 429 | } |
| 430 | |
| 431 | OveruseFrameDetectorResourceAdaptationModule:: |
Evan Shrubsole | aa6fbc1 | 2020-02-25 15:26:01 | [diff] [blame] | 432 | ~OveruseFrameDetectorResourceAdaptationModule() { |
| 433 | RTC_DCHECK_EQ(state_, State::kStopped); |
| 434 | } |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 435 | |
Henrik Boström | a3d4252 | 2020-01-16 12:55:29 | [diff] [blame] | 436 | void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( |
Henrik Boström | d238200 | 2020-01-10 14:44:01 | [diff] [blame] | 437 | ResourceAdaptationModuleListener* adaptation_listener) { |
Evan Shrubsole | aa6fbc1 | 2020-02-25 15:26:01 | [diff] [blame] | 438 | RTC_DCHECK_EQ(state_, State::kStopped); |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 439 | RTC_DCHECK(encoder_settings_.has_value()); |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 440 | // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener |
| 441 | // should be passed in and why. If resources are separated from modules then |
| 442 | // those resources may be started or stopped separately from the module. |
Henrik Boström | d238200 | 2020-01-10 14:44:01 | [diff] [blame] | 443 | RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 444 | encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions()); |
Evan Shrubsole | aa6fbc1 | 2020-02-25 15:26:01 | [diff] [blame] | 445 | for (auto& resource_and_reason : resources_) { |
| 446 | resource_and_reason.resource->RegisterListener(this); |
| 447 | } |
| 448 | state_ = State::kStarted; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 449 | } |
| 450 | |
Henrik Boström | a3d4252 | 2020-01-16 12:55:29 | [diff] [blame] | 451 | void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 452 | encode_usage_resource_->StopCheckForOveruse(); |
| 453 | quality_scaler_resource_->StopCheckForOveruse(); |
Evan Shrubsole | aa6fbc1 | 2020-02-25 15:26:01 | [diff] [blame] | 454 | for (auto& resource_and_reason : resources_) { |
| 455 | resource_and_reason.resource->UnregisterListener(this); |
| 456 | } |
| 457 | state_ = State::kStopped; |
| 458 | } |
| 459 | |
| 460 | void OveruseFrameDetectorResourceAdaptationModule::AddResource( |
| 461 | Resource* resource) { |
| 462 | return AddResource(resource, AdaptationObserverInterface::AdaptReason::kCpu); |
| 463 | } |
| 464 | |
| 465 | void OveruseFrameDetectorResourceAdaptationModule::AddResource( |
| 466 | Resource* resource, |
| 467 | AdaptationObserverInterface::AdaptReason reason) { |
| 468 | RTC_DCHECK(resource); |
| 469 | RTC_DCHECK(absl::c_find_if(resources_, |
| 470 | [resource](const ResourceAndReason& r) { |
| 471 | return r.resource == resource; |
| 472 | }) == resources_.end()) |
| 473 | << "Resource " << resource->name() << " already was inserted"; |
| 474 | resources_.emplace_back(resource, reason); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 475 | } |
| 476 | |
Henrik Boström | a3d4252 | 2020-01-16 12:55:29 | [diff] [blame] | 477 | void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( |
| 478 | bool has_input_video) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 479 | // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS. |
Henrik Boström | a3d4252 | 2020-01-16 12:55:29 | [diff] [blame] | 480 | has_input_video_ = has_input_video; |
| 481 | } |
| 482 | |
| 483 | void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( |
| 484 | DegradationPreference degradation_preference) { |
| 485 | if (degradation_preference_ != degradation_preference) { |
| 486 | // Reset adaptation state, so that we're not tricked into thinking there's |
| 487 | // an already pending request of the same type. |
| 488 | last_adaptation_request_.reset(); |
| 489 | if (degradation_preference == DegradationPreference::BALANCED || |
| 490 | degradation_preference_ == DegradationPreference::BALANCED) { |
| 491 | // TODO(asapersson): Consider removing |adapt_counters_| map and use one |
| 492 | // AdaptCounter for all modes. |
| 493 | source_restrictor_->ClearRestrictions(); |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 494 | ClearAdaptCounters(); |
Henrik Boström | a3d4252 | 2020-01-16 12:55:29 | [diff] [blame] | 495 | } |
| 496 | } |
| 497 | degradation_preference_ = degradation_preference; |
| 498 | MaybeUpdateVideoSourceRestrictions(); |
| 499 | } |
| 500 | |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 501 | void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings( |
| 502 | EncoderSettings encoder_settings) { |
| 503 | encoder_settings_ = std::move(encoder_settings); |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 504 | |
| 505 | quality_rampup_experiment_.SetMaxBitrate( |
| 506 | LastInputFrameSizeOrDefault(), |
| 507 | encoder_settings_->video_codec().maxBitrate); |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 508 | MaybeUpdateTargetFrameRate(); |
| 509 | } |
| 510 | |
Evan Shrubsole | 7c3a1fc | 2020-02-04 15:26:38 | [diff] [blame] | 511 | void OveruseFrameDetectorResourceAdaptationModule::SetStartBitrate( |
| 512 | DataRate start_bitrate) { |
| 513 | if (!start_bitrate.IsZero()) |
| 514 | encoder_target_bitrate_bps_ = start_bitrate.bps(); |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 515 | initial_frame_dropper_->SetStartBitrate(start_bitrate, |
| 516 | clock_->TimeInMicroseconds()); |
Evan Shrubsole | 7c3a1fc | 2020-02-04 15:26:38 | [diff] [blame] | 517 | } |
| 518 | |
| 519 | void OveruseFrameDetectorResourceAdaptationModule::SetTargetBitrate( |
| 520 | DataRate target_bitrate) { |
| 521 | if (!target_bitrate.IsZero()) |
| 522 | encoder_target_bitrate_bps_ = target_bitrate.bps(); |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 523 | initial_frame_dropper_->SetTargetBitrate(target_bitrate, |
| 524 | clock_->TimeInMilliseconds()); |
Henrik Boström | ede69c0 | 2020-01-21 16:45:35 | [diff] [blame] | 525 | } |
| 526 | |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 527 | void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates( |
| 528 | const VideoEncoder::RateControlParameters& encoder_rates) { |
| 529 | encoder_rates_ = encoder_rates; |
| 530 | } |
| 531 | |
Henrik Boström | fae6f0e | 2020-01-20 10:16:50 | [diff] [blame] | 532 | void OveruseFrameDetectorResourceAdaptationModule:: |
| 533 | ResetVideoSourceRestrictions() { |
| 534 | last_adaptation_request_.reset(); |
| 535 | source_restrictor_->ClearRestrictions(); |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 536 | ClearAdaptCounters(); |
Henrik Boström | fae6f0e | 2020-01-20 10:16:50 | [diff] [blame] | 537 | MaybeUpdateVideoSourceRestrictions(); |
| 538 | } |
| 539 | |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 540 | void OveruseFrameDetectorResourceAdaptationModule::OnFrame( |
| 541 | const VideoFrame& frame) { |
| 542 | last_input_frame_size_ = frame.size(); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 543 | } |
| 544 | |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 545 | void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { |
Evan Shrubsole | 6c13fd9 | 2020-01-21 08:57:54 | [diff] [blame] | 546 | int fps_count = GetConstAdaptCounter().FramerateCount( |
| 547 | AdaptationObserverInterface::AdaptReason::kQuality); |
| 548 | int res_count = GetConstAdaptCounter().ResolutionCount( |
| 549 | AdaptationObserverInterface::AdaptReason::kQuality); |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 550 | OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); |
Evan Shrubsole | 6c13fd9 | 2020-01-21 08:57:54 | [diff] [blame] | 551 | if (degradation_preference() == DegradationPreference::BALANCED && |
| 552 | GetConstAdaptCounter().FramerateCount( |
| 553 | AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) { |
| 554 | // Adapt framerate in same step as resolution. |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 555 | OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); |
Evan Shrubsole | 6c13fd9 | 2020-01-21 08:57:54 | [diff] [blame] | 556 | } |
| 557 | if (GetConstAdaptCounter().ResolutionCount( |
| 558 | AdaptationObserverInterface::AdaptReason::kQuality) > res_count) { |
| 559 | encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); |
| 560 | } |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 561 | initial_frame_dropper_->OnFrameDroppedDueToSize(); |
Evan Shrubsole | 6c13fd9 | 2020-01-21 08:57:54 | [diff] [blame] | 562 | } |
| 563 | |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 564 | void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted( |
| 565 | const VideoFrame& cropped_frame, |
| 566 | int64_t time_when_first_seen_us) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 567 | encode_usage_resource_->OnEncodeStarted(cropped_frame, |
| 568 | time_when_first_seen_us); |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 569 | } |
| 570 | |
| 571 | void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted( |
Evan Shrubsole | bfe3ef8 | 2020-01-30 13:29:35 | [diff] [blame] | 572 | const EncodedImage& encoded_image, |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 573 | int64_t time_sent_in_us, |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 574 | absl::optional<int> encode_duration_us) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 575 | // Inform |encode_usage_resource_| of the encode completed event. |
Evan Shrubsole | bfe3ef8 | 2020-01-30 13:29:35 | [diff] [blame] | 576 | uint32_t timestamp = encoded_image.Timestamp(); |
| 577 | int64_t capture_time_us = |
| 578 | encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 579 | encode_usage_resource_->OnEncodeCompleted( |
| 580 | timestamp, time_sent_in_us, capture_time_us, encode_duration_us); |
| 581 | // Inform |quality_scaler_resource_| of the encode completed event. |
| 582 | quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 583 | } |
| 584 | |
Evan Shrubsole | c809e8b | 2020-01-31 14:36:35 | [diff] [blame] | 585 | void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped( |
| 586 | EncodedImageCallback::DropReason reason) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 587 | quality_scaler_resource_->OnFrameDropped(reason); |
Evan Shrubsole | c809e8b | 2020-01-31 14:36:35 | [diff] [blame] | 588 | } |
| 589 | |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 590 | bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const { |
| 591 | return initial_frame_dropper_->DropInitialFrames(); |
Evan Shrubsole | f2be3ef | 2020-02-03 09:43:31 | [diff] [blame] | 592 | } |
| 593 | |
Mirko Bonadei | 2e161c4 | 2020-02-20 08:45:01 | [diff] [blame] | 594 | void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() { |
| 595 | initial_frame_dropper_->OnMaybeEncodeFrame(); |
| 596 | MaybePerformQualityRampupExperiment(); |
| 597 | } |
| 598 | |
Evan Shrubsole | cf05952 | 2020-01-29 16:04:44 | [diff] [blame] | 599 | void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( |
| 600 | absl::optional<VideoEncoder::QpThresholds> qp_thresholds) { |
| 601 | if (qp_thresholds.has_value()) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 602 | quality_scaler_resource_->StopCheckForOveruse(); |
| 603 | quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value()); |
Evan Shrubsole | cf05952 | 2020-01-29 16:04:44 | [diff] [blame] | 604 | } else { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 605 | quality_scaler_resource_->StopCheckForOveruse(); |
Evan Shrubsole | cf05952 | 2020-01-29 16:04:44 | [diff] [blame] | 606 | } |
Henrik Boström | 8cfecac | 2020-02-07 10:29:14 | [diff] [blame] | 607 | initial_frame_dropper_->OnQualityScalerSettingsUpdated(); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 608 | } |
| 609 | |
Evan Shrubsole | c81798b | 2020-02-03 14:46:08 | [diff] [blame] | 610 | void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( |
| 611 | const VideoEncoder::EncoderInfo& encoder_info) { |
| 612 | const auto scaling_settings = encoder_info.scaling_settings; |
| 613 | const bool quality_scaling_allowed = |
| 614 | IsResolutionScalingEnabled(degradation_preference_) && |
| 615 | scaling_settings.thresholds; |
| 616 | |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 617 | // TODO(https://crbug.com/webrtc/11222): Should this move to |
| 618 | // QualityScalerResource? |
Evan Shrubsole | c81798b | 2020-02-03 14:46:08 | [diff] [blame] | 619 | if (quality_scaling_allowed) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 620 | if (!quality_scaler_resource_->is_started()) { |
Evan Shrubsole | c81798b | 2020-02-03 14:46:08 | [diff] [blame] | 621 | // Quality scaler has not already been configured. |
| 622 | |
| 623 | // Use experimental thresholds if available. |
| 624 | absl::optional<VideoEncoder::QpThresholds> experimental_thresholds; |
| 625 | if (quality_scaling_experiment_enabled_) { |
| 626 | experimental_thresholds = QualityScalingExperiment::GetQpThresholds( |
| 627 | GetVideoCodecTypeOrGeneric()); |
| 628 | } |
| 629 | UpdateQualityScalerSettings(experimental_thresholds |
| 630 | ? *experimental_thresholds |
| 631 | : *(scaling_settings.thresholds)); |
| 632 | } |
| 633 | } else { |
| 634 | UpdateQualityScalerSettings(absl::nullopt); |
| 635 | } |
| 636 | |
| 637 | // Set the qp-thresholds to the balanced settings if balanced mode. |
| 638 | if (degradation_preference_ == DegradationPreference::BALANCED && |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 639 | quality_scaler_resource_->is_started()) { |
Evan Shrubsole | c81798b | 2020-02-03 14:46:08 | [diff] [blame] | 640 | absl::optional<VideoEncoder::QpThresholds> thresholds = |
| 641 | balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(), |
| 642 | LastInputFrameSizeOrDefault()); |
| 643 | if (thresholds) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 644 | quality_scaler_resource_->SetQpThresholds(*thresholds); |
Evan Shrubsole | c81798b | 2020-02-03 14:46:08 | [diff] [blame] | 645 | } |
| 646 | } |
| 647 | |
| 648 | encoder_stats_observer_->OnAdaptationChanged( |
| 649 | VideoStreamEncoderObserver::AdaptationReason::kNone, |
| 650 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), |
| 651 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); |
| 652 | } |
| 653 | |
Henrik Boström | 48258ac | 2020-02-06 11:49:57 | [diff] [blame] | 654 | ResourceListenerResponse |
| 655 | OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured( |
| 656 | const Resource& resource) { |
Evan Shrubsole | aa6fbc1 | 2020-02-25 15:26:01 | [diff] [blame] | 657 | const auto& registered_resource = |
| 658 | absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) { |
| 659 | return r.resource == &resource; |
| 660 | }); |
| 661 | RTC_DCHECK(registered_resource != resources_.end()) |
| 662 | << resource.name() << " not found."; |
| 663 | |
| 664 | const AdaptationObserverInterface::AdaptReason reason = |
| 665 | registered_resource->reason; |
Henrik Boström | 48258ac | 2020-02-06 11:49:57 | [diff] [blame] | 666 | switch (resource.usage_state()) { |
| 667 | case ResourceUsageState::kOveruse: |
| 668 | return OnResourceOveruse(reason); |
| 669 | case ResourceUsageState::kStable: |
| 670 | // Do nothing. |
| 671 | // |
| 672 | // This module has two resources: |encoude_usage_resource_| and |
| 673 | // |quality_scaler_resource_|. A smarter adaptation module might not |
| 674 | // attempt to adapt up unless ALL resources were underused, but this |
| 675 | // module acts on each resource's measurement in isolation - without |
| 676 | // taking the current usage of any other resource into account. |
| 677 | return ResourceListenerResponse::kNothing; |
| 678 | case ResourceUsageState::kUnderuse: |
| 679 | OnResourceUnderuse(reason); |
| 680 | return ResourceListenerResponse::kNothing; |
| 681 | } |
| 682 | } |
| 683 | |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 684 | absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget> |
| 685 | OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget( |
| 686 | int input_pixels, |
| 687 | int input_fps, |
| 688 | AdaptationObserverInterface::AdaptReason reason) const { |
| 689 | // Preconditions for being able to adapt up: |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 690 | if (!has_input_video_) |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 691 | return absl::nullopt; |
| 692 | // 1. We can't adapt up if we're already at the highest setting. |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 693 | int num_downgrades = GetConstAdaptCounter().TotalCount(reason); |
| 694 | RTC_DCHECK_GE(num_downgrades, 0); |
| 695 | if (num_downgrades == 0) |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 696 | return absl::nullopt; |
| 697 | // 2. We shouldn't adapt up if we're currently waiting for a previous upgrade |
| 698 | // to have an effect. |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 699 | // TODO(hbos): What about in the case of other degradation preferences? |
| 700 | bool last_adaptation_was_up = |
| 701 | last_adaptation_request_ && |
| 702 | last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; |
| 703 | if (last_adaptation_was_up && |
| 704 | degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 705 | input_pixels <= last_adaptation_request_->input_pixel_count_) { |
| 706 | return absl::nullopt; |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 707 | } |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 708 | // 3. We shouldn't adapt up if BalancedSettings doesn't allow it, which is |
| 709 | // only applicable if reason is kQuality and preference is BALANCED. |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 710 | if (reason == AdaptationObserverInterface::AdaptReason::kQuality && |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 711 | EffectiveDegradationPreference() == DegradationPreference::BALANCED && |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 712 | !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(), input_pixels, |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 713 | encoder_target_bitrate_bps_.value_or(0))) { |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 714 | return absl::nullopt; |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 715 | } |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 716 | |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 717 | // Attempt to find an allowed adaptation target. |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 718 | switch (EffectiveDegradationPreference()) { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 719 | case DegradationPreference::BALANCED: { |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 720 | // Attempt to increase target frame rate. |
| 721 | int target_fps = |
| 722 | balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(), input_pixels); |
| 723 | if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) { |
| 724 | return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, |
| 725 | target_fps); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 726 | } |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 727 | // Fall-through to maybe-adapting resolution, unless |balanced_settings_| |
| 728 | // forbids it based on bitrate. |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 729 | if (reason == AdaptationObserverInterface::AdaptReason::kQuality && |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 730 | !balanced_settings_.CanAdaptUpResolution( |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 731 | GetVideoCodecTypeOrGeneric(), input_pixels, |
Evan Shrubsole | 7c3a1fc | 2020-02-04 15:26:38 | [diff] [blame] | 732 | encoder_target_bitrate_bps_.value_or(0))) { |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 733 | return absl::nullopt; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 734 | } |
| 735 | // Scale up resolution. |
Danil Chapovalov | 64f1f3f | 2020-01-16 13:41:10 | [diff] [blame] | 736 | ABSL_FALLTHROUGH_INTENDED; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 737 | } |
| 738 | case DegradationPreference::MAINTAIN_FRAMERATE: { |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 739 | // Don't adapt resolution if CanAdaptUpResolution() forbids it based on |
| 740 | // bitrate and limits specified by encoder capabilities. |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 741 | if (reason == AdaptationObserverInterface::AdaptReason::kQuality && |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 742 | !CanAdaptUpResolution(input_pixels, |
Evan Shrubsole | 7c3a1fc | 2020-02-04 15:26:38 | [diff] [blame] | 743 | encoder_target_bitrate_bps_.value_or(0))) { |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 744 | return absl::nullopt; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 745 | } |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 746 | // Attempt to increase pixel count. |
| 747 | int target_pixels = input_pixels; |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 748 | if (GetConstAdaptCounter().ResolutionCount() == 1) { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 749 | RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 750 | target_pixels = std::numeric_limits<int>::max(); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 751 | } |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 752 | target_pixels = |
| 753 | VideoSourceRestrictor::GetHigherResolutionThan(target_pixels); |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 754 | if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels)) |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 755 | return absl::nullopt; |
| 756 | return AdaptationTarget(AdaptationAction::kIncreaseResolution, |
| 757 | target_pixels); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 758 | } |
| 759 | case DegradationPreference::MAINTAIN_RESOLUTION: { |
| 760 | // Scale up framerate. |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 761 | int target_fps = input_fps; |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 762 | if (GetConstAdaptCounter().FramerateCount() == 1) { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 763 | RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 764 | target_fps = std::numeric_limits<int>::max(); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 765 | } |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 766 | target_fps = VideoSourceRestrictor::GetHigherFrameRateThan(target_fps); |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 767 | if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps)) |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 768 | return absl::nullopt; |
| 769 | return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 770 | } |
| 771 | case DegradationPreference::DISABLED: |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 772 | return absl::nullopt; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 773 | } |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 774 | } |
| 775 | |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 776 | absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget> |
| 777 | OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget( |
| 778 | int input_pixels, |
| 779 | int input_fps, |
| 780 | int min_pixels_per_frame, |
| 781 | AdaptationObserverInterface::AdaptReason reason) const { |
| 782 | // Preconditions for being able to adapt down: |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 783 | if (!has_input_video_) |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 784 | return absl::nullopt; |
| 785 | // 1. We are not disabled. |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 786 | // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it |
| 787 | // causes scaling due to bandwidth constraints (QualityScalerResource) to be |
| 788 | // ignored, not just CPU signals. This is not a use case we want to support; |
| 789 | // remove the enum value. |
| 790 | if (degradation_preference_ == DegradationPreference::DISABLED) |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 791 | return absl::nullopt; |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 792 | bool last_adaptation_was_down = |
| 793 | last_adaptation_request_ && |
| 794 | last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 795 | // 2. We shouldn't adapt down if our frame rate is below the minimum or if its |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 796 | // currently unknown. |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 797 | if (EffectiveDegradationPreference() == |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 798 | DegradationPreference::MAINTAIN_RESOLUTION) { |
| 799 | // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake |
| 800 | // - delete it. |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 801 | if (input_fps <= 0 || |
| 802 | (last_adaptation_was_down && input_fps < kMinFramerateFps)) { |
| 803 | return absl::nullopt; |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 804 | } |
| 805 | } |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 806 | // 3. We shouldn't adapt down if we're currently waiting for a previous |
| 807 | // downgrade to have an effect. |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 808 | // TODO(hbos): What about in the case of other degradation preferences? |
| 809 | if (last_adaptation_was_down && |
| 810 | degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 811 | input_pixels >= last_adaptation_request_->input_pixel_count_) { |
| 812 | return absl::nullopt; |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 813 | } |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 814 | |
| 815 | // Attempt to find an allowed adaptation target. |
| 816 | switch (EffectiveDegradationPreference()) { |
| 817 | case DegradationPreference::BALANCED: { |
| 818 | // Try scale down framerate, if lower. |
| 819 | int target_fps = |
| 820 | balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(), input_pixels); |
| 821 | if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { |
| 822 | return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, |
| 823 | target_fps); |
| 824 | } |
| 825 | // Scale down resolution. |
| 826 | ABSL_FALLTHROUGH_INTENDED; |
| 827 | } |
| 828 | case DegradationPreference::MAINTAIN_FRAMERATE: { |
| 829 | // Scale down resolution. |
| 830 | int target_pixels = |
| 831 | VideoSourceRestrictor::GetLowerResolutionThan(input_pixels); |
| 832 | // TODO(https://crbug.com/webrtc/11222): Move this logic to |
| 833 | // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation |
| 834 | // target is available should not have side-effects. |
| 835 | if (target_pixels < min_pixels_per_frame) |
| 836 | encoder_stats_observer_->OnMinPixelLimitReached(); |
| 837 | if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels, |
| 838 | min_pixels_per_frame)) { |
| 839 | return absl::nullopt; |
| 840 | } |
| 841 | return AdaptationTarget(AdaptationAction::kDecreaseResolution, |
| 842 | target_pixels); |
| 843 | } |
| 844 | case DegradationPreference::MAINTAIN_RESOLUTION: { |
| 845 | int target_fps = VideoSourceRestrictor::GetLowerFrameRateThan(input_fps); |
| 846 | if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) |
| 847 | return absl::nullopt; |
| 848 | return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps); |
| 849 | } |
| 850 | case DegradationPreference::DISABLED: |
| 851 | RTC_NOTREACHED(); |
| 852 | return absl::nullopt; |
| 853 | } |
| 854 | } |
| 855 | |
| 856 | void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget( |
| 857 | const AdaptationTarget& target, |
| 858 | int min_pixels_per_frame, |
| 859 | AdaptationObserverInterface::AdaptReason reason) { |
| 860 | switch (target.action) { |
| 861 | case AdaptationAction::kIncreaseResolution: |
| 862 | source_restrictor_->IncreaseResolutionTo(target.value); |
| 863 | GetAdaptCounter().DecrementResolution(reason); |
| 864 | return; |
| 865 | case AdaptationAction::kDecreaseResolution: |
| 866 | source_restrictor_->DecreaseResolutionTo(target.value, |
| 867 | min_pixels_per_frame); |
| 868 | GetAdaptCounter().IncrementResolution(reason); |
| 869 | return; |
| 870 | case AdaptationAction::kIncreaseFrameRate: |
| 871 | source_restrictor_->IncreaseFrameRateTo(target.value); |
| 872 | GetAdaptCounter().DecrementFramerate(reason, target.value); |
| 873 | // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps. |
| 874 | // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic |
| 875 | // in DecrementFramerate() makes it hard to predict whether this will be |
| 876 | // the last step. Remove the dependency on GetConstAdaptCounter(). |
| 877 | if (EffectiveDegradationPreference() == DegradationPreference::BALANCED && |
| 878 | GetConstAdaptCounter().FramerateCount() == 0 && |
| 879 | target.value != std::numeric_limits<int>::max()) { |
| 880 | RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; |
| 881 | source_restrictor_->IncreaseFrameRateTo( |
| 882 | std::numeric_limits<int>::max()); |
| 883 | } |
| 884 | return; |
| 885 | case AdaptationAction::kDecreaseFrameRate: |
| 886 | source_restrictor_->DecreaseFrameRateTo(target.value); |
| 887 | GetAdaptCounter().IncrementFramerate(reason); |
| 888 | return; |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( |
| 893 | AdaptationObserverInterface::AdaptReason reason) { |
| 894 | int input_pixels = LastInputFrameSizeOrDefault(); |
| 895 | int input_fps = encoder_stats_observer_->GetInputFrameRate(); |
| 896 | int min_pixels_per_frame = MinPixelsPerFrame(); |
| 897 | // Should we adapt, if so to what target? |
| 898 | absl::optional<AdaptationTarget> target = |
| 899 | GetAdaptUpTarget(input_pixels, input_fps, reason); |
| 900 | if (!target.has_value()) |
| 901 | return; |
| 902 | // Apply target. |
| 903 | ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason); |
| 904 | last_adaptation_request_.emplace(AdaptationRequest{ |
| 905 | input_pixels, input_fps, AdaptationRequest::Mode::kAdaptUp}); |
| 906 | // Update VideoSourceRestrictions based on adaptation. This also informs the |
| 907 | // |adaptation_listener_|. |
| 908 | MaybeUpdateVideoSourceRestrictions(); |
| 909 | // Stats and logging. |
| 910 | UpdateAdaptationStats(reason); |
| 911 | RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString(); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 912 | } |
| 913 | |
Henrik Boström | 48258ac | 2020-02-06 11:49:57 | [diff] [blame] | 914 | ResourceListenerResponse |
| 915 | OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 916 | AdaptationObserverInterface::AdaptReason reason) { |
Henrik Boström | a3d4252 | 2020-01-16 12:55:29 | [diff] [blame] | 917 | if (!has_input_video_) |
Henrik Boström | 48258ac | 2020-02-06 11:49:57 | [diff] [blame] | 918 | return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 919 | int input_pixels = LastInputFrameSizeOrDefault(); |
| 920 | int input_fps = encoder_stats_observer_->GetInputFrameRate(); |
| 921 | int min_pixels_per_frame = MinPixelsPerFrame(); |
| 922 | // Should we adapt, if so to what target? |
| 923 | absl::optional<AdaptationTarget> target = |
| 924 | GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame, reason); |
| 925 | if (!target.has_value()) |
Henrik Boström | d2a1f09b | 2020-02-25 08:46:36 | [diff] [blame] | 926 | return ResourceListenerResponse::kNothing; |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 927 | // Apply target. |
| 928 | ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason); |
| 929 | last_adaptation_request_.emplace(AdaptationRequest{ |
| 930 | input_pixels, input_fps, AdaptationRequest::Mode::kAdaptDown}); |
| 931 | // Update VideoSourceRestrictions based on adaptation. This also informs the |
| 932 | // |adaptation_listener_|. |
Henrik Boström | 07b17df | 2020-01-15 10:42:12 | [diff] [blame] | 933 | MaybeUpdateVideoSourceRestrictions(); |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 934 | // Stats and logging. |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 935 | UpdateAdaptationStats(reason); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 936 | RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString(); |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 937 | // In BALANCED, if requested FPS is higher or close to input FPS to the target |
| 938 | // we tell the QualityScaler to increase its frequency. |
| 939 | if (EffectiveDegradationPreference() == DegradationPreference::BALANCED && |
| 940 | target->action == AdaptationAction::kDecreaseFrameRate) { |
| 941 | absl::optional<int> min_diff = balanced_settings_.MinFpsDiff(input_pixels); |
| 942 | if (min_diff && input_fps > 0) { |
| 943 | int fps_diff = input_fps - target->value; |
| 944 | if (fps_diff < min_diff.value()) { |
| 945 | return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; |
| 946 | } |
| 947 | } |
| 948 | } |
| 949 | return ResourceListenerResponse::kNothing; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 950 | } |
| 951 | |
Henrik Boström | ad515a2 | 2020-01-27 12:38:05 | [diff] [blame] | 952 | // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle |
| 953 | // pipelining encoders better (multiple input frames before something comes |
| 954 | // out). This should effectively turn off CPU adaptations for systems that |
| 955 | // remotely cope with the load right now. |
| 956 | CpuOveruseOptions |
| 957 | OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const { |
| 958 | // This is already ensured by the only caller of this method: |
| 959 | // StartResourceAdaptation(). |
| 960 | RTC_DCHECK(encoder_settings_.has_value()); |
| 961 | CpuOveruseOptions options; |
| 962 | // Hardware accelerated encoders are assumed to be pipelined; give them |
| 963 | // additional overuse time. |
| 964 | if (encoder_settings_->encoder_info().is_hardware_accelerated) { |
| 965 | options.low_encode_usage_threshold_percent = 150; |
| 966 | options.high_encode_usage_threshold_percent = 200; |
| 967 | } |
| 968 | if (experiment_cpu_load_estimator_) { |
| 969 | options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec; |
| 970 | } |
| 971 | return options; |
| 972 | } |
| 973 | |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 974 | VideoCodecType |
| 975 | OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric() |
| 976 | const { |
| 977 | return encoder_settings_.has_value() |
| 978 | ? encoder_settings_->encoder_config().codec_type |
| 979 | : kVideoCodecGeneric; |
| 980 | } |
| 981 | |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 982 | int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault() |
| 983 | const { |
| 984 | // The dependency on this hardcoded resolution is inherited from old code, |
| 985 | // which used this resolution as a stand-in for not knowing the resolution |
| 986 | // yet. |
| 987 | // TODO(hbos): Can we simply DCHECK has_value() before usage instead? Having a |
| 988 | // DCHECK passed all the tests but adding it does change the requirements of |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 989 | // this class (= not being allowed to call OnResourceUnderuse() or |
| 990 | // OnResourceOveruse() before OnFrame()) and deserves a standalone CL. |
Henrik Boström | d4578ae | 2020-01-22 15:16:04 | [diff] [blame] | 991 | return last_input_frame_size_.value_or( |
| 992 | VideoStreamEncoder::kDefaultLastFrameInfoWidth * |
| 993 | VideoStreamEncoder::kDefaultLastFrameInfoHeight); |
| 994 | } |
| 995 | |
Henrik Boström | 6038383 | 2020-02-28 08:03:53 | [diff] [blame^] | 996 | int OveruseFrameDetectorResourceAdaptationModule::MinPixelsPerFrame() const { |
| 997 | return encoder_settings_.has_value() |
| 998 | ? encoder_settings_->encoder_info() |
| 999 | .scaling_settings.min_pixels_per_frame |
| 1000 | : kDefaultMinPixelsPerFrame; |
| 1001 | } |
| 1002 | |
Henrik Boström | 8234b92 | 2020-01-13 16:26:50 | [diff] [blame] | 1003 | void OveruseFrameDetectorResourceAdaptationModule:: |
Henrik Boström | 07b17df | 2020-01-15 10:42:12 | [diff] [blame] | 1004 | MaybeUpdateVideoSourceRestrictions() { |
| 1005 | VideoSourceRestrictions new_restrictions = ApplyDegradationPreference( |
| 1006 | source_restrictor_->source_restrictions(), degradation_preference_); |
| 1007 | if (video_source_restrictions_ != new_restrictions) { |
| 1008 | video_source_restrictions_ = std::move(new_restrictions); |
Henrik Boström | 8234b92 | 2020-01-13 16:26:50 | [diff] [blame] | 1009 | adaptation_listener_->OnVideoSourceRestrictionsUpdated( |
Henrik Boström | 07b17df | 2020-01-15 10:42:12 | [diff] [blame] | 1010 | video_source_restrictions_); |
Henrik Boström | fae6f0e | 2020-01-20 10:16:50 | [diff] [blame] | 1011 | MaybeUpdateTargetFrameRate(); |
| 1012 | } |
| 1013 | } |
| 1014 | |
| 1015 | void OveruseFrameDetectorResourceAdaptationModule:: |
| 1016 | MaybeUpdateTargetFrameRate() { |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 1017 | absl::optional<double> codec_max_frame_rate = |
| 1018 | encoder_settings_.has_value() |
| 1019 | ? absl::optional<double>( |
| 1020 | encoder_settings_->video_codec().maxFramerate) |
| 1021 | : absl::nullopt; |
Henrik Boström | fae6f0e | 2020-01-20 10:16:50 | [diff] [blame] | 1022 | // The current target framerate is the maximum frame rate as specified by |
| 1023 | // the current codec configuration or any limit imposed by the adaptation |
| 1024 | // module. This is used to make sure overuse detection doesn't needlessly |
| 1025 | // trigger in low and/or variable framerate scenarios. |
| 1026 | absl::optional<double> target_frame_rate = |
| 1027 | ApplyDegradationPreference(source_restrictor_->source_restrictions(), |
| 1028 | degradation_preference_) |
| 1029 | .max_frame_rate(); |
| 1030 | if (!target_frame_rate.has_value() || |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 1031 | (codec_max_frame_rate.has_value() && |
| 1032 | codec_max_frame_rate.value() < target_frame_rate.value())) { |
| 1033 | target_frame_rate = codec_max_frame_rate; |
Henrik Boström | fae6f0e | 2020-01-20 10:16:50 | [diff] [blame] | 1034 | } |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1035 | encode_usage_resource_->SetTargetFrameRate(target_frame_rate); |
Henrik Boström | 8234b92 | 2020-01-13 16:26:50 | [diff] [blame] | 1036 | } |
| 1037 | |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1038 | // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. |
| 1039 | void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1040 | AdaptationObserverInterface::AdaptReason reason) { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1041 | switch (reason) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1042 | case AdaptationObserverInterface::AdaptReason::kCpu: |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1043 | encoder_stats_observer_->OnAdaptationChanged( |
| 1044 | VideoStreamEncoderObserver::AdaptationReason::kCpu, |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1045 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), |
| 1046 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1047 | break; |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1048 | case AdaptationObserverInterface::AdaptReason::kQuality: |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1049 | encoder_stats_observer_->OnAdaptationChanged( |
| 1050 | VideoStreamEncoderObserver::AdaptationReason::kQuality, |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1051 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), |
| 1052 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1053 | break; |
| 1054 | } |
| 1055 | } |
| 1056 | |
| 1057 | VideoStreamEncoderObserver::AdaptationSteps |
| 1058 | OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1059 | AdaptationObserverInterface::AdaptReason reason) { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1060 | VideoStreamEncoderObserver::AdaptationSteps counts = |
| 1061 | GetConstAdaptCounter().Counts(reason); |
| 1062 | switch (reason) { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1063 | case AdaptationObserverInterface::AdaptReason::kCpu: |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1064 | if (!IsFramerateScalingEnabled(degradation_preference_)) |
| 1065 | counts.num_framerate_reductions = absl::nullopt; |
| 1066 | if (!IsResolutionScalingEnabled(degradation_preference_)) |
| 1067 | counts.num_resolution_reductions = absl::nullopt; |
| 1068 | break; |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1069 | case AdaptationObserverInterface::AdaptReason::kQuality: |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1070 | if (!IsFramerateScalingEnabled(degradation_preference_) || |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1071 | !quality_scaler_resource_->is_started()) { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1072 | counts.num_framerate_reductions = absl::nullopt; |
| 1073 | } |
| 1074 | if (!IsResolutionScalingEnabled(degradation_preference_) || |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1075 | !quality_scaler_resource_->is_started()) { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1076 | counts.num_resolution_reductions = absl::nullopt; |
| 1077 | } |
| 1078 | break; |
| 1079 | } |
| 1080 | return counts; |
| 1081 | } |
| 1082 | |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 1083 | DegradationPreference |
| 1084 | OveruseFrameDetectorResourceAdaptationModule::EffectiveDegradationPreference() |
| 1085 | const { |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1086 | // Balanced mode for screenshare works via automatic animation detection: |
| 1087 | // Resolution is capped for fullscreen animated content. |
| 1088 | // Adapatation is done only via framerate downgrade. |
| 1089 | // Thus effective degradation preference is MAINTAIN_RESOLUTION. |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 1090 | return (encoder_settings_.has_value() && |
| 1091 | encoder_settings_->encoder_config().content_type == |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1092 | VideoEncoderConfig::ContentType::kScreen && |
| 1093 | degradation_preference_ == DegradationPreference::BALANCED) |
| 1094 | ? DegradationPreference::MAINTAIN_RESOLUTION |
| 1095 | : degradation_preference_; |
| 1096 | } |
| 1097 | |
| 1098 | OveruseFrameDetectorResourceAdaptationModule::AdaptCounter& |
| 1099 | OveruseFrameDetectorResourceAdaptationModule::GetAdaptCounter() { |
| 1100 | return adapt_counters_[degradation_preference_]; |
| 1101 | } |
| 1102 | |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 1103 | void OveruseFrameDetectorResourceAdaptationModule::ClearAdaptCounters() { |
| 1104 | adapt_counters_.clear(); |
| 1105 | adapt_counters_.insert( |
| 1106 | std::make_pair(DegradationPreference::DISABLED, AdaptCounter())); |
| 1107 | adapt_counters_.insert(std::make_pair( |
| 1108 | DegradationPreference::MAINTAIN_FRAMERATE, AdaptCounter())); |
| 1109 | adapt_counters_.insert(std::make_pair( |
| 1110 | DegradationPreference::MAINTAIN_RESOLUTION, AdaptCounter())); |
| 1111 | adapt_counters_.insert( |
| 1112 | std::make_pair(DegradationPreference::BALANCED, AdaptCounter())); |
| 1113 | } |
| 1114 | |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1115 | const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter& |
Henrik Boström | d6fb409 | 2020-02-25 16:51:08 | [diff] [blame] | 1116 | OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() const { |
| 1117 | auto it = adapt_counters_.find(degradation_preference_); |
| 1118 | RTC_DCHECK(it != adapt_counters_.cend()); |
| 1119 | return it->second; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1120 | } |
| 1121 | |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1122 | bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( |
| 1123 | int pixels, |
| 1124 | uint32_t bitrate_bps) const { |
| 1125 | absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits = |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 1126 | encoder_settings_.has_value() |
| 1127 | ? GetEncoderBitrateLimits( |
| 1128 | encoder_settings_->encoder_info(), |
Henrik Boström | 02956fe | 2020-02-25 08:35:03 | [diff] [blame] | 1129 | VideoSourceRestrictor::GetHigherResolutionThan(pixels)) |
Henrik Boström | 4bab2fc | 2020-01-21 10:18:06 | [diff] [blame] | 1130 | : absl::nullopt; |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1131 | if (!bitrate_limits.has_value() || bitrate_bps == 0) { |
| 1132 | return true; // No limit configured or bitrate provided. |
| 1133 | } |
| 1134 | RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels); |
| 1135 | return bitrate_bps >= |
| 1136 | static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps); |
| 1137 | } |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 1138 | void OveruseFrameDetectorResourceAdaptationModule:: |
| 1139 | MaybePerformQualityRampupExperiment() { |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1140 | if (!quality_scaler_resource_->is_started()) |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 1141 | return; |
| 1142 | |
| 1143 | if (quality_rampup_done_) |
| 1144 | return; |
| 1145 | |
| 1146 | int64_t now_ms = clock_->TimeInMilliseconds(); |
| 1147 | uint32_t bw_kbps = encoder_rates_.has_value() |
| 1148 | ? encoder_rates_.value().bandwidth_allocation.kbps() |
| 1149 | : 0; |
| 1150 | |
| 1151 | bool try_quality_rampup = false; |
| 1152 | if (quality_rampup_experiment_.BwHigh(now_ms, bw_kbps)) { |
| 1153 | // Verify that encoder is at max bitrate and the QP is low. |
| 1154 | if (encoder_settings_ && |
| 1155 | encoder_target_bitrate_bps_.value_or(0) == |
| 1156 | encoder_settings_->video_codec().maxBitrate * 1000 && |
Henrik Boström | 7875c99 | 2020-02-06 09:35:00 | [diff] [blame] | 1157 | quality_scaler_resource_->QpFastFilterLow()) { |
Evan Shrubsole | e331a12 | 2020-02-05 12:30:33 | [diff] [blame] | 1158 | try_quality_rampup = true; |
| 1159 | } |
| 1160 | } |
| 1161 | if (try_quality_rampup && |
| 1162 | GetConstAdaptCounter().ResolutionCount( |
| 1163 | AdaptationObserverInterface::AdaptReason::kQuality) > 0 && |
| 1164 | GetConstAdaptCounter().TotalCount( |
| 1165 | AdaptationObserverInterface::AdaptReason::kCpu) == 0) { |
| 1166 | RTC_LOG(LS_INFO) << "Reset quality limitations."; |
| 1167 | ResetVideoSourceRestrictions(); |
| 1168 | quality_rampup_done_ = true; |
| 1169 | } |
| 1170 | } |
| 1171 | |
Henrik Boström | b08882b | 2020-01-07 09:11:17 | [diff] [blame] | 1172 | } // namespace webrtc |