Refactor out VideoStreamEncoder's overuse logic to separate module.
This CL puts the VideoStreamEncoder's current adaptation logic inside
the new class OveruseFrameDetectorResourceAdaptationModule. The
intention is not to change any behavior, only to move code.
Future CLs should step by step decrease the coupling between
OveruseFrameDetectorResourceAdaptationModule, VideoStreamEncoder and
the VideoStreamEncoder's QualityScaler by introducing more abstract
interfaces. This is not done in this CL because it is large enough as
it is, but the long term goal is to make it possible to replace the
existing overuse module with a different implementation.
This CL relies on existing tests exercising the VideoStreamEncoder, but
part of making overuse logic modular should include testing each module
separately as well as continued integration testing of the
VideoStreamEncoder.
Bug: webrtc:11222
Change-Id: I316a174adfd00d60cdd224a23a5f616efd235d13
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/161953
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Cr-Commit-Position: refs/heads/master@{#30163}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 0f9829e..51cd40b 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -184,11 +184,14 @@
"frame_encode_metadata_writer.h",
"overuse_frame_detector.cc",
"overuse_frame_detector.h",
+ "overuse_frame_detector_resource_adaptation_module.cc",
+ "overuse_frame_detector_resource_adaptation_module.h",
"video_stream_encoder.cc",
"video_stream_encoder.h",
]
deps = [
+ "../api:rtp_parameters",
"../api/units:data_rate",
"../api/video:encoded_image",
"../api/video:video_bitrate_allocation",
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
new file mode 100644
index 0000000..b5c61b2
--- /dev/null
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -0,0 +1,900 @@
+/*
+ * Copyright 2019 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/overuse_frame_detector_resource_adaptation_module.h"
+
+#include <algorithm>
+#include <limits>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "absl/algorithm/container.h"
+#include "api/video/video_source_interface.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/strings/string_builder.h"
+#include "rtc_base/system/fallthrough.h"
+#include "video/video_stream_encoder.h"
+
+namespace webrtc {
+
+namespace {
+
+const int kMinFramerateFps = 2;
+
+bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
+ return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
+ degradation_preference == DegradationPreference::BALANCED;
+}
+
+bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
+ return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION ||
+ degradation_preference == DegradationPreference::BALANCED;
+}
+
+} // namespace
+
+// VideoSourceProxy is responsible ensuring thread safety between calls to
+// OveruseFrameDetectorResourceAdaptationModule::SetSource that will happen on
+// libjingle's worker thread when a video capturer is connected to the encoder
+// and the encoder task queue (encoder_queue_) where the encoder reports its
+// VideoSinkWants.
+class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy {
+ public:
+ explicit VideoSourceProxy(rtc::VideoSinkInterface<VideoFrame>* sink)
+ : sink_(sink),
+ degradation_preference_(DegradationPreference::DISABLED),
+ source_(nullptr),
+ max_framerate_(std::numeric_limits<int>::max()),
+ max_pixels_(std::numeric_limits<int>::max()),
+ resolution_alignment_(1) {}
+
+ void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
+ const DegradationPreference& degradation_preference) {
+ // Called on libjingle's worker thread.
+ RTC_DCHECK_RUN_ON(&main_checker_);
+ rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr;
+ rtc::VideoSinkWants wants;
+ {
+ rtc::CritScope lock(&crit_);
+ degradation_preference_ = degradation_preference;
+ old_source = source_;
+ source_ = source;
+ wants = GetActiveSinkWantsInternal();
+ }
+
+ if (old_source != source && old_source != nullptr) {
+ old_source->RemoveSink(sink_);
+ }
+
+ if (!source) {
+ return;
+ }
+
+ source->AddOrUpdateSink(sink_, wants);
+ }
+
+ void SetMaxFramerateAndAlignment(int max_framerate,
+ int resolution_alignment) {
+ RTC_DCHECK_GT(max_framerate, 0);
+ rtc::CritScope lock(&crit_);
+ if (max_framerate == max_framerate_ &&
+ resolution_alignment == resolution_alignment_) {
+ return;
+ }
+
+ RTC_LOG(LS_INFO) << "Set max framerate: " << max_framerate
+ << " and resolution alignment: " << resolution_alignment;
+ max_framerate_ = max_framerate;
+ resolution_alignment_ = resolution_alignment;
+ if (source_) {
+ source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal());
+ }
+ }
+
+ void SetWantsRotationApplied(bool rotation_applied) {
+ rtc::CritScope lock(&crit_);
+ sink_wants_.rotation_applied = rotation_applied;
+ if (source_) {
+ source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal());
+ }
+ }
+
+ rtc::VideoSinkWants GetActiveSinkWants() {
+ rtc::CritScope lock(&crit_);
+ return GetActiveSinkWantsInternal();
+ }
+
+ void ResetPixelFpsCount() {
+ rtc::CritScope lock(&crit_);
+ sink_wants_.max_pixel_count = std::numeric_limits<int>::max();
+ sink_wants_.target_pixel_count.reset();
+ sink_wants_.max_framerate_fps = std::numeric_limits<int>::max();
+ if (source_)
+ source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal());
+ }
+
+ bool RequestResolutionLowerThan(int pixel_count,
+ int min_pixels_per_frame,
+ bool* min_pixels_reached) {
+ // Called on the encoder task queue.
+ rtc::CritScope lock(&crit_);
+ if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
+ // This can happen since |degradation_preference_| is set on libjingle's
+ // worker thread but the adaptation is done on the encoder task queue.
+ return false;
+ }
+ // The input video frame size will have a resolution less than or equal to
+ // |max_pixel_count| depending on how the source can scale the frame size.
+ const int pixels_wanted = (pixel_count * 3) / 5;
+ if (pixels_wanted >= sink_wants_.max_pixel_count) {
+ return false;
+ }
+ if (pixels_wanted < min_pixels_per_frame) {
+ *min_pixels_reached = true;
+ return false;
+ }
+ RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: "
+ << pixels_wanted;
+ sink_wants_.max_pixel_count = pixels_wanted;
+ sink_wants_.target_pixel_count = absl::nullopt;
+ source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal());
+ return true;
+ }
+
+ int RequestFramerateLowerThan(int fps) {
+ // Called on the encoder task queue.
+ // The input video frame rate will be scaled down to 2/3, rounding down.
+ int framerate_wanted = (fps * 2) / 3;
+ return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
+ }
+
+ int GetHigherResolutionThan(int pixel_count) const {
+ // On step down we request at most 3/5 the pixel count of the previous
+ // resolution, so in order to take "one step up" we request a resolution
+ // as close as possible to 5/3 of the current resolution. The actual pixel
+ // count selected depends on the capabilities of the source. In order to
+ // not take a too large step up, we cap the requested pixel count to be at
+ // most four time the current number of pixels.
+ return (pixel_count * 5) / 3;
+ }
+
+ bool RequestHigherResolutionThan(int pixel_count) {
+ // Called on the encoder task queue.
+ rtc::CritScope lock(&crit_);
+ if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
+ // This can happen since |degradation_preference_| is set on libjingle's
+ // worker thread but the adaptation is done on the encoder task queue.
+ return false;
+ }
+ int max_pixels_wanted = pixel_count;
+ if (max_pixels_wanted != std::numeric_limits<int>::max())
+ max_pixels_wanted = pixel_count * 4;
+
+ if (max_pixels_wanted <= sink_wants_.max_pixel_count)
+ return false;
+
+ sink_wants_.max_pixel_count = max_pixels_wanted;
+ if (max_pixels_wanted == std::numeric_limits<int>::max()) {
+ // Remove any constraints.
+ sink_wants_.target_pixel_count.reset();
+ } else {
+ sink_wants_.target_pixel_count = GetHigherResolutionThan(pixel_count);
+ }
+ RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
+ << max_pixels_wanted;
+ source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal());
+ return true;
+ }
+
+ // Request upgrade in framerate. Returns the new requested frame, or -1 if
+ // no change requested. Note that maxint may be returned if limits due to
+ // adaptation requests are removed completely. In that case, consider
+ // |max_framerate_| to be the current limit (assuming the capturer complies).
+ int RequestHigherFramerateThan(int fps) {
+ // Called on the encoder task queue.
+ // The input frame rate will be scaled up to the last step, with rounding.
+ int framerate_wanted = fps;
+ if (fps != std::numeric_limits<int>::max())
+ framerate_wanted = (fps * 3) / 2;
+
+ return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1;
+ }
+
+ bool RestrictFramerate(int fps) {
+ // Called on the encoder task queue.
+ rtc::CritScope lock(&crit_);
+ if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
+ return false;
+
+ const int fps_wanted = std::max(kMinFramerateFps, fps);
+ if (fps_wanted >= sink_wants_.max_framerate_fps)
+ return false;
+
+ RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted;
+ sink_wants_.max_framerate_fps = fps_wanted;
+ source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal());
+ return true;
+ }
+
+ bool IncreaseFramerate(int fps) {
+ // Called on the encoder task queue.
+ rtc::CritScope lock(&crit_);
+ if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
+ return false;
+
+ const int fps_wanted = std::max(kMinFramerateFps, fps);
+ if (fps_wanted <= sink_wants_.max_framerate_fps)
+ return false;
+
+ RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted;
+ sink_wants_.max_framerate_fps = fps_wanted;
+ source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal());
+ return true;
+ }
+
+ // Used in automatic animation detection for screenshare.
+ bool RestrictPixels(int max_pixels) {
+ // Called on the encoder task queue.
+ rtc::CritScope lock(&crit_);
+ if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
+ // This can happen since |degradation_preference_| is set on libjingle's
+ // worker thread but the adaptation is done on the encoder task queue.
+ return false;
+ }
+ max_pixels_ = max_pixels;
+ RTC_LOG(LS_INFO) << "Applying max pixel restriction: " << max_pixels;
+ source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal());
+ return true;
+ }
+
+ private:
+ rtc::VideoSinkWants GetActiveSinkWantsInternal()
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
+ rtc::VideoSinkWants wants = sink_wants_;
+ // Clear any constraints from the current sink wants that don't apply to
+ // the used degradation_preference.
+ switch (degradation_preference_) {
+ case DegradationPreference::BALANCED:
+ break;
+ case DegradationPreference::MAINTAIN_FRAMERATE:
+ wants.max_framerate_fps = std::numeric_limits<int>::max();
+ break;
+ case DegradationPreference::MAINTAIN_RESOLUTION:
+ wants.max_pixel_count = std::numeric_limits<int>::max();
+ wants.target_pixel_count.reset();
+ break;
+ case DegradationPreference::DISABLED:
+ wants.max_pixel_count = std::numeric_limits<int>::max();
+ wants.target_pixel_count.reset();
+ wants.max_framerate_fps = std::numeric_limits<int>::max();
+ }
+ // Limit to configured max framerate.
+ wants.max_framerate_fps = std::min(max_framerate_, wants.max_framerate_fps);
+ // Limit resolution due to automatic animation detection for screenshare.
+ wants.max_pixel_count = std::min(max_pixels_, wants.max_pixel_count);
+ wants.resolution_alignment = resolution_alignment_;
+
+ return wants;
+ }
+
+ rtc::CriticalSection crit_;
+ SequenceChecker main_checker_;
+ rtc::VideoSinkInterface<VideoFrame>* const sink_;
+ rtc::VideoSinkWants sink_wants_ RTC_GUARDED_BY(&crit_);
+ DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_);
+ rtc::VideoSourceInterface<VideoFrame>* source_ RTC_GUARDED_BY(&crit_);
+ int max_framerate_ RTC_GUARDED_BY(&crit_);
+ int max_pixels_ RTC_GUARDED_BY(&crit_);
+ int resolution_alignment_ RTC_GUARDED_BY(&crit_);
+
+ RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
+};
+
+// Class holding adaptation information.
+OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::AdaptCounter() {
+ fps_counters_.resize(kScaleReasonSize);
+ resolution_counters_.resize(kScaleReasonSize);
+ static_assert(kScaleReasonSize == 2, "Update MoveCount.");
+}
+
+OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::~AdaptCounter() {}
+
+std::string
+OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ToString() const {
+ rtc::StringBuilder ss;
+ ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
+ ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
+ return ss.Release();
+}
+
+VideoStreamEncoderObserver::AdaptationSteps
+OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::Counts(
+ int reason) const {
+ VideoStreamEncoderObserver::AdaptationSteps counts;
+ counts.num_framerate_reductions = fps_counters_[reason];
+ counts.num_resolution_reductions = resolution_counters_[reason];
+ return counts;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::
+ IncrementFramerate(int reason) {
+ ++(fps_counters_[reason]);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::
+ IncrementResolution(int reason) {
+ ++(resolution_counters_[reason]);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::
+ DecrementFramerate(int reason) {
+ if (fps_counters_[reason] == 0) {
+ // Balanced mode: Adapt up is in a different order, switch reason.
+ // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3).
+ // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0}
+ // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0}
+ // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0}
+ // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0}
+ RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
+ RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
+ MoveCount(&resolution_counters_, reason);
+ MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize);
+ }
+ --(fps_counters_[reason]);
+ RTC_DCHECK_GE(fps_counters_[reason], 0);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::
+ DecrementResolution(int reason) {
+ if (resolution_counters_[reason] == 0) {
+ // Balanced mode: Adapt up is in a different order, switch reason.
+ RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
+ RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
+ MoveCount(&fps_counters_, reason);
+ MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize);
+ }
+ --(resolution_counters_[reason]);
+ RTC_DCHECK_GE(resolution_counters_[reason], 0);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::
+ DecrementFramerate(int reason, int cur_fps) {
+ DecrementFramerate(reason);
+ // Reset if at max fps (i.e. in case of fewer steps up than down).
+ if (cur_fps == std::numeric_limits<int>::max())
+ absl::c_fill(fps_counters_, 0);
+}
+
+int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::FramerateCount()
+ const {
+ return Count(fps_counters_);
+}
+
+int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::
+ ResolutionCount() const {
+ return Count(resolution_counters_);
+}
+
+int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::FramerateCount(
+ int reason) const {
+ return fps_counters_[reason];
+}
+
+int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ResolutionCount(
+ int reason) const {
+ return resolution_counters_[reason];
+}
+
+int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::TotalCount(
+ int reason) const {
+ return FramerateCount(reason) + ResolutionCount(reason);
+}
+
+int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::Count(
+ const std::vector<int>& counters) const {
+ return absl::c_accumulate(counters, 0);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::MoveCount(
+ std::vector<int>* counters,
+ int from_reason) {
+ int to_reason = (from_reason + 1) % kScaleReasonSize;
+ ++((*counters)[to_reason]);
+ --((*counters)[from_reason]);
+}
+
+std::string
+OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ToString(
+ const std::vector<int>& counters) const {
+ rtc::StringBuilder ss;
+ for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
+ ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
+ }
+ return ss.Release();
+}
+
+OveruseFrameDetectorResourceAdaptationModule::
+ OveruseFrameDetectorResourceAdaptationModule(
+ rtc::VideoSinkInterface<VideoFrame>* sink,
+ std::unique_ptr<OveruseFrameDetector> overuse_detector,
+ VideoStreamEncoderObserver* encoder_stats_observer)
+ : encoder_queue_(nullptr),
+ degradation_preference_(DegradationPreference::DISABLED),
+ adapt_counters_(),
+ balanced_settings_(),
+ last_adaptation_request_(absl::nullopt),
+ last_frame_pixel_count_(absl::nullopt),
+ source_proxy_(std::make_unique<VideoSourceProxy>(sink)),
+ overuse_detector_(std::move(overuse_detector)),
+ codec_max_framerate_(-1),
+ encoder_start_bitrate_bps_(0),
+ is_quality_scaler_enabled_(false),
+ encoder_config_(),
+ encoder_(nullptr),
+ encoder_stats_observer_(encoder_stats_observer) {
+ RTC_DCHECK(overuse_detector_);
+ RTC_DCHECK(encoder_stats_observer_);
+}
+
+OveruseFrameDetectorResourceAdaptationModule::
+ ~OveruseFrameDetectorResourceAdaptationModule() {}
+
+void OveruseFrameDetectorResourceAdaptationModule::Initialize(
+ rtc::TaskQueue* encoder_queue) {
+ RTC_DCHECK(!encoder_queue_);
+ encoder_queue_ = encoder_queue;
+ RTC_DCHECK(encoder_queue_);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::SetEncoder(
+ VideoEncoder* encoder) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ encoder_ = encoder;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse(
+ const CpuOveruseOptions& options) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ RTC_DCHECK(encoder_);
+ overuse_detector_->StartCheckForOveruse(encoder_queue_, options, this);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ overuse_detector_->StopCheckForOveruse();
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured(
+ const VideoFrame& frame,
+ int64_t time_when_first_seen_us) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ overuse_detector_->FrameCaptured(frame, time_when_first_seen_us);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::FrameSent(
+ uint32_t timestamp,
+ int64_t time_sent_in_us,
+ int64_t capture_time_us,
+ absl::optional<int> encode_duration_us) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us,
+ encode_duration_us);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::SetLastFramePixelCount(
+ absl::optional<int> last_frame_pixel_count) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ last_frame_pixel_count_ = last_frame_pixel_count;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig(
+ VideoEncoderConfig encoder_config) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ encoder_config_ = std::move(encoder_config);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFramerate(
+ int codec_max_framerate) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ codec_max_framerate_ = codec_max_framerate;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps(
+ uint32_t encoder_start_bitrate_bps) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ encoder_start_bitrate_bps_ = encoder_start_bitrate_bps;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::SetIsQualityScalerEnabled(
+ bool is_quality_scaler_enabled) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ is_quality_scaler_enabled_ = is_quality_scaler_enabled;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::SetSource(
+ rtc::VideoSourceInterface<VideoFrame>* source,
+ const DegradationPreference& degradation_preference) {
+ source_proxy_->SetSource(source, degradation_preference);
+ encoder_queue_->PostTask([this, degradation_preference] {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ if (degradation_preference_ != degradation_preference) {
+ // Reset adaptation state, so that we're not tricked into thinking there's
+ // an already pending request of the same type.
+ last_adaptation_request_.reset();
+ if (degradation_preference == DegradationPreference::BALANCED ||
+ degradation_preference_ == DegradationPreference::BALANCED) {
+ // TODO(asapersson): Consider removing |adapt_counters_| map and use one
+ // AdaptCounter for all modes.
+ source_proxy_->ResetPixelFpsCount();
+ adapt_counters_.clear();
+ }
+ }
+ degradation_preference_ = degradation_preference;
+ });
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::
+ SetSourceWantsRotationApplied(bool rotation_applied) {
+ source_proxy_->SetWantsRotationApplied(rotation_applied);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::
+ SetSourceMaxFramerateAndAlignment(int max_framerate,
+ int resolution_alignment) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ source_proxy_->SetMaxFramerateAndAlignment(max_framerate,
+ resolution_alignment);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::SetSourceMaxPixels(
+ int max_pixels) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ source_proxy_->RestrictPixels(max_pixels);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ // Get the current target framerate, ie the maximum framerate as specified by
+ // the current codec configuration, or any limit imposed by cpu adaption in
+ // maintain-resolution or balanced mode. This is used to make sure overuse
+ // detection doesn't needlessly trigger in low and/or variable framerate
+ // scenarios.
+ int target_framerate =
+ std::min(codec_max_framerate_,
+ source_proxy_->GetActiveSinkWants().max_framerate_fps);
+ overuse_detector_->OnTargetFramerateUpdated(target_framerate);
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::ResetAdaptationCounters() {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ last_adaptation_request_.reset();
+ source_proxy_->ResetPixelFpsCount();
+ adapt_counters_.clear();
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ const AdaptCounter& adapt_counter = GetConstAdaptCounter();
+ int num_downgrades = adapt_counter.TotalCount(reason);
+ if (num_downgrades == 0)
+ return;
+ RTC_DCHECK_GT(num_downgrades, 0);
+
+ AdaptationRequest adaptation_request = {
+ *last_frame_pixel_count_, encoder_stats_observer_->GetInputFrameRate(),
+ AdaptationRequest::Mode::kAdaptUp};
+
+ bool adapt_up_requested =
+ last_adaptation_request_ &&
+ last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
+
+ if (EffectiveDegradataionPreference() ==
+ DegradationPreference::MAINTAIN_FRAMERATE) {
+ if (adapt_up_requested &&
+ adaptation_request.input_pixel_count_ <=
+ last_adaptation_request_->input_pixel_count_) {
+ // Don't request higher resolution if the current resolution is not
+ // higher than the last time we asked for the resolution to be higher.
+ return;
+ }
+ }
+
+ switch (EffectiveDegradataionPreference()) {
+ case DegradationPreference::BALANCED: {
+ // Check if quality should be increased based on bitrate.
+ if (reason == kQuality &&
+ !balanced_settings_.CanAdaptUp(*last_frame_pixel_count_,
+ encoder_start_bitrate_bps_)) {
+ return;
+ }
+ // Try scale up framerate, if higher.
+ int fps = balanced_settings_.MaxFps(encoder_config_.codec_type,
+ *last_frame_pixel_count_);
+ if (source_proxy_->IncreaseFramerate(fps)) {
+ GetAdaptCounter().DecrementFramerate(reason, fps);
+ // Reset framerate in case of fewer fps steps down than up.
+ if (adapt_counter.FramerateCount() == 0 &&
+ fps != std::numeric_limits<int>::max()) {
+ RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
+ source_proxy_->IncreaseFramerate(std::numeric_limits<int>::max());
+ }
+ break;
+ }
+ // Check if resolution should be increased based on bitrate.
+ if (reason == kQuality &&
+ !balanced_settings_.CanAdaptUpResolution(
+ *last_frame_pixel_count_, encoder_start_bitrate_bps_)) {
+ return;
+ }
+ // Scale up resolution.
+ RTC_FALLTHROUGH();
+ }
+ case DegradationPreference::MAINTAIN_FRAMERATE: {
+ // Check if resolution should be increased based on bitrate and
+ // limits specified by encoder capabilities.
+ if (reason == kQuality &&
+ !CanAdaptUpResolution(*last_frame_pixel_count_,
+ encoder_start_bitrate_bps_)) {
+ return;
+ }
+
+ // Scale up resolution.
+ int pixel_count = adaptation_request.input_pixel_count_;
+ if (adapt_counter.ResolutionCount() == 1) {
+ RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
+ pixel_count = std::numeric_limits<int>::max();
+ }
+ if (!source_proxy_->RequestHigherResolutionThan(pixel_count))
+ return;
+ GetAdaptCounter().DecrementResolution(reason);
+ break;
+ }
+ case DegradationPreference::MAINTAIN_RESOLUTION: {
+ // Scale up framerate.
+ int fps = adaptation_request.framerate_fps_;
+ if (adapt_counter.FramerateCount() == 1) {
+ RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
+ fps = std::numeric_limits<int>::max();
+ }
+
+ const int requested_framerate =
+ source_proxy_->RequestHigherFramerateThan(fps);
+ if (requested_framerate == -1) {
+ overuse_detector_->OnTargetFramerateUpdated(codec_max_framerate_);
+ return;
+ }
+ overuse_detector_->OnTargetFramerateUpdated(
+ std::min(codec_max_framerate_, requested_framerate));
+ GetAdaptCounter().DecrementFramerate(reason);
+ break;
+ }
+ case DegradationPreference::DISABLED:
+ return;
+ }
+
+ last_adaptation_request_.emplace(adaptation_request);
+
+ UpdateAdaptationStats(reason);
+
+ RTC_LOG(LS_INFO) << adapt_counter.ToString();
+}
+
+bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
+ AdaptReason reason) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ AdaptationRequest adaptation_request = {
+ *last_frame_pixel_count_, encoder_stats_observer_->GetInputFrameRate(),
+ AdaptationRequest::Mode::kAdaptDown};
+
+ bool downgrade_requested =
+ last_adaptation_request_ &&
+ last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
+
+ bool did_adapt = true;
+
+ switch (EffectiveDegradataionPreference()) {
+ case DegradationPreference::BALANCED:
+ break;
+ case DegradationPreference::MAINTAIN_FRAMERATE:
+ if (downgrade_requested &&
+ adaptation_request.input_pixel_count_ >=
+ last_adaptation_request_->input_pixel_count_) {
+ // Don't request lower resolution if the current resolution is not
+ // lower than the last time we asked for the resolution to be lowered.
+ return true;
+ }
+ break;
+ case DegradationPreference::MAINTAIN_RESOLUTION:
+ if (adaptation_request.framerate_fps_ <= 0 ||
+ (downgrade_requested &&
+ adaptation_request.framerate_fps_ < kMinFramerateFps)) {
+ // If no input fps estimate available, can't determine how to scale down
+ // framerate. Otherwise, don't request lower framerate if we don't have
+ // a valid frame rate. Since framerate, unlike resolution, is a measure
+ // we have to estimate, and can fluctuate naturally over time, don't
+ // make the same kind of limitations as for resolution, but trust the
+ // overuse detector to not trigger too often.
+ return true;
+ }
+ break;
+ case DegradationPreference::DISABLED:
+ return true;
+ }
+
+ switch (EffectiveDegradataionPreference()) {
+ case DegradationPreference::BALANCED: {
+ // Try scale down framerate, if lower.
+ int fps = balanced_settings_.MinFps(encoder_config_.codec_type,
+ *last_frame_pixel_count_);
+ if (source_proxy_->RestrictFramerate(fps)) {
+ GetAdaptCounter().IncrementFramerate(reason);
+ // Check if requested fps is higher (or close to) input fps.
+ absl::optional<int> min_diff =
+ balanced_settings_.MinFpsDiff(*last_frame_pixel_count_);
+ if (min_diff && adaptation_request.framerate_fps_ > 0) {
+ int fps_diff = adaptation_request.framerate_fps_ - fps;
+ if (fps_diff < min_diff.value()) {
+ did_adapt = false;
+ }
+ }
+ break;
+ }
+ // Scale down resolution.
+ RTC_FALLTHROUGH();
+ }
+ case DegradationPreference::MAINTAIN_FRAMERATE: {
+ // Scale down resolution.
+ bool min_pixels_reached = false;
+ if (!source_proxy_->RequestResolutionLowerThan(
+ adaptation_request.input_pixel_count_,
+ encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame,
+ &min_pixels_reached)) {
+ if (min_pixels_reached)
+ encoder_stats_observer_->OnMinPixelLimitReached();
+ return true;
+ }
+ GetAdaptCounter().IncrementResolution(reason);
+ break;
+ }
+ case DegradationPreference::MAINTAIN_RESOLUTION: {
+ // Scale down framerate.
+ const int requested_framerate = source_proxy_->RequestFramerateLowerThan(
+ adaptation_request.framerate_fps_);
+ if (requested_framerate == -1)
+ return true;
+ RTC_DCHECK_NE(codec_max_framerate_, -1);
+ overuse_detector_->OnTargetFramerateUpdated(
+ std::min(codec_max_framerate_, requested_framerate));
+ GetAdaptCounter().IncrementFramerate(reason);
+ break;
+ }
+ case DegradationPreference::DISABLED:
+ RTC_NOTREACHED();
+ }
+
+ last_adaptation_request_.emplace(adaptation_request);
+
+ UpdateAdaptationStats(reason);
+
+ RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString();
+ return did_adapt;
+}
+
+// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
+void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats(
+ AdaptReason reason) {
+ switch (reason) {
+ case kCpu:
+ encoder_stats_observer_->OnAdaptationChanged(
+ VideoStreamEncoderObserver::AdaptationReason::kCpu,
+ GetActiveCounts(kCpu), GetActiveCounts(kQuality));
+ break;
+ case kQuality:
+ encoder_stats_observer_->OnAdaptationChanged(
+ VideoStreamEncoderObserver::AdaptationReason::kQuality,
+ GetActiveCounts(kCpu), GetActiveCounts(kQuality));
+ break;
+ }
+}
+
+VideoStreamEncoderObserver::AdaptationSteps
+OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
+ AdaptReason reason) {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ VideoStreamEncoderObserver::AdaptationSteps counts =
+ GetConstAdaptCounter().Counts(reason);
+ switch (reason) {
+ case kCpu:
+ if (!IsFramerateScalingEnabled(degradation_preference_))
+ counts.num_framerate_reductions = absl::nullopt;
+ if (!IsResolutionScalingEnabled(degradation_preference_))
+ counts.num_resolution_reductions = absl::nullopt;
+ break;
+ case kQuality:
+ if (!IsFramerateScalingEnabled(degradation_preference_) ||
+ !is_quality_scaler_enabled_) {
+ counts.num_framerate_reductions = absl::nullopt;
+ }
+ if (!IsResolutionScalingEnabled(degradation_preference_) ||
+ !is_quality_scaler_enabled_) {
+ counts.num_resolution_reductions = absl::nullopt;
+ }
+ break;
+ }
+ return counts;
+}
+
+DegradationPreference OveruseFrameDetectorResourceAdaptationModule::
+ EffectiveDegradataionPreference() {
+ // Balanced mode for screenshare works via automatic animation detection:
+ // Resolution is capped for fullscreen animated content.
+ // Adapatation is done only via framerate downgrade.
+ // Thus effective degradation preference is MAINTAIN_RESOLUTION.
+ return (encoder_config_.content_type ==
+ VideoEncoderConfig::ContentType::kScreen &&
+ degradation_preference_ == DegradationPreference::BALANCED)
+ ? DegradationPreference::MAINTAIN_RESOLUTION
+ : degradation_preference_;
+}
+
+OveruseFrameDetectorResourceAdaptationModule::AdaptCounter&
+OveruseFrameDetectorResourceAdaptationModule::GetAdaptCounter() {
+ return adapt_counters_[degradation_preference_];
+}
+
+const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter&
+OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ return adapt_counters_[degradation_preference_];
+}
+
+absl::optional<VideoEncoder::QpThresholds>
+OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ RTC_DCHECK(last_frame_pixel_count_.has_value());
+ return balanced_settings_.GetQpThresholds(encoder_config_.codec_type,
+ last_frame_pixel_count_.value());
+}
+
+bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
+ int pixels,
+ uint32_t bitrate_bps) const {
+ absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
+ GetEncoderBitrateLimits(encoder_->GetEncoderInfo(),
+ source_proxy_->GetHigherResolutionThan(pixels));
+ if (!bitrate_limits.has_value() || bitrate_bps == 0) {
+ return true; // No limit configured or bitrate provided.
+ }
+ RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels);
+ return bitrate_bps >=
+ static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
+}
+
+} // namespace webrtc
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
new file mode 100644
index 0000000..d13a6a7
--- /dev/null
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -0,0 +1,221 @@
+/*
+ * Copyright 2019 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_
+#define VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/rtp_parameters.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_sink_interface.h"
+#include "api/video/video_source_interface.h"
+#include "api/video/video_stream_encoder_observer.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/video_encoder_config.h"
+#include "rtc_base/experiments/balanced_degradation_settings.h"
+#include "video/overuse_frame_detector.h"
+
+namespace webrtc {
+
+// This class is used by the VideoStreamEncoder and is responsible for adapting
+// resolution up or down based on encode usage percent. It keeps track of video
+// source settings, adaptation counters and may get influenced by
+// VideoStreamEncoder's quality scaler through AdaptUp() and AdaptDown() calls.
+// TODO(hbos): Reduce the coupling with VideoStreamEncoder.
+// TODO(hbos): Add unittests specific to this class, it is currently only tested
+// indirectly in video_stream_encoder_unittest.cc and other tests exercising
+// VideoStreamEncoder.
+// TODO(hbos): Create and implement an abstract interface
+// ResourceAdaptationModuleInterface and make this class inherit it. Use the
+// generic interface in VideoStreamEncoder, unblocking other modules from being
+// implemented and used.
+class OveruseFrameDetectorResourceAdaptationModule
+ : public AdaptationObserverInterface {
+ public:
+ OveruseFrameDetectorResourceAdaptationModule(
+ rtc::VideoSinkInterface<VideoFrame>* sink,
+ std::unique_ptr<OveruseFrameDetector> overuse_detector,
+ VideoStreamEncoderObserver* encoder_stats_observer);
+ ~OveruseFrameDetectorResourceAdaptationModule() override;
+
+ void Initialize(rtc::TaskQueue* encoder_queue);
+ // Sets the encoder to reconfigure based on overuse.
+ // TODO(hbos): Don't reconfigure the encoder directly. Instead, define the
+ // output of a resource adaptation module as a struct and let the
+ // VideoStreamEncoder handle the interaction with the actual encoder.
+ void SetEncoder(VideoEncoder* encoder);
+
+ DegradationPreference degradation_preference() const {
+ RTC_DCHECK(encoder_queue_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ return degradation_preference_;
+ }
+
+ // Input to the OveruseFrameDetector, which are required for this module to
+ // function. These map to OveruseFrameDetector methods.
+ // TODO(hbos): Define virtual methods in ResourceAdaptationModuleInterface
+ // for input that are more generic so that this class can be used without
+ // assumptions about underlying implementation.
+ void StartCheckForOveruse(const CpuOveruseOptions& options);
+ void StopCheckForOveruse();
+ void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us);
+ void FrameSent(uint32_t timestamp,
+ int64_t time_sent_in_us,
+ int64_t capture_time_us,
+ absl::optional<int> encode_duration_us);
+
+ // Various other settings and feedback mechanisms.
+ // TODO(hbos): Find a common interface that would make sense for a generic
+ // resource adaptation module. Unify code paths where possible. Do we really
+ // need this many public methods?
+ void SetLastFramePixelCount(absl::optional<int> last_frame_pixel_count);
+ void SetEncoderConfig(VideoEncoderConfig encoder_config);
+ void SetCodecMaxFramerate(int codec_max_framerate);
+ void SetEncoderStartBitrateBps(uint32_t encoder_start_bitrate_bps);
+ // Inform the detector whether or not the quality scaler is enabled. This
+ // helps GetActiveCounts() return absl::nullopt when appropriate.
+ // TODO(hbos): This feels really hacky, can we report the right values without
+ // this boolean? It would be really easy to report the wrong thing if this
+ // method is called incorrectly.
+ void SetIsQualityScalerEnabled(bool is_quality_scaler_enabled);
+
+ void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
+ const DegradationPreference& degradation_preference);
+ void SetSourceWantsRotationApplied(bool rotation_applied);
+ void SetSourceMaxFramerateAndAlignment(int max_framerate,
+ int resolution_alignment);
+ void SetSourceMaxPixels(int max_pixels);
+
+ // TODO(hbos): Can we get rid of this? Seems we should know whether the frame
+ // rate has updated.
+ void RefreshTargetFramerate();
+ void ResetAdaptationCounters();
+
+ class AdaptCounter final {
+ public:
+ AdaptCounter();
+ ~AdaptCounter();
+
+ // Get number of adaptation downscales for |reason|.
+ VideoStreamEncoderObserver::AdaptationSteps Counts(int reason) const;
+
+ std::string ToString() const;
+
+ void IncrementFramerate(int reason);
+ void IncrementResolution(int reason);
+ void DecrementFramerate(int reason);
+ void DecrementResolution(int reason);
+ void DecrementFramerate(int reason, int cur_fps);
+
+ // Gets the total number of downgrades (for all adapt reasons).
+ int FramerateCount() const;
+ int ResolutionCount() const;
+
+ // Gets the total number of downgrades for |reason|.
+ int FramerateCount(int reason) const;
+ int ResolutionCount(int reason) const;
+ int TotalCount(int reason) const;
+
+ private:
+ std::string ToString(const std::vector<int>& counters) const;
+ int Count(const std::vector<int>& counters) const;
+ void MoveCount(std::vector<int>* counters, int from_reason);
+
+ // Degradation counters holding number of framerate/resolution reductions
+ // per adapt reason.
+ std::vector<int> fps_counters_;
+ std::vector<int> resolution_counters_;
+ };
+
+ // AdaptationObserverInterface implementation. Used both "internally" as
+ // feedback from |overuse_detector_|, and externally from VideoStreamEncoder:
+ // - It is wired to the VideoStreamEncoder::quality_scaler_.
+ // - It is invoked by VideoStreamEncoder::MaybeEncodeVideoFrame().
+ // TODO(hbos): Decouple quality scaling and resource adaptation, or find an
+ // interface for reconfiguring externally.
+ // TODO(hbos): VideoStreamEncoder should not be responsible for any part of
+ // the adaptation.
+ void AdaptUp(AdaptReason reason) override;
+ bool AdaptDown(AdaptReason reason) override;
+
+ // Used by VideoStreamEncoder when ConfigureQualityScaler() occurs and the
+ // |encoder_stats_observer_| is called outside of this class.
+ // TODO(hbos): Decouple quality scaling and resource adaptation logic and make
+ // this method private.
+ VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
+ AdaptReason reason);
+
+ // Used by VideoStreamEncoder::MaybeEncodeVideoFrame().
+ // TODO(hbos): VideoStreamEncoder should not be responsible for any part of
+ // the adaptation. Move this logic to this module?
+ const AdaptCounter& GetConstAdaptCounter();
+
+ // Used by VideoStreamEncoder::ConfigureQualityScaler().
+ // TODO(hbos): Decouple quality scaling and resource adaptation logic and
+ // delete this method.
+ absl::optional<VideoEncoder::QpThresholds> GetQpThresholds() const;
+
+ private:
+ class VideoSourceProxy;
+
+ struct AdaptationRequest {
+ // The pixel count produced by the source at the time of the adaptation.
+ int input_pixel_count_;
+ // Framerate received from the source at the time of the adaptation.
+ int framerate_fps_;
+ // Indicates if request was to adapt up or down.
+ enum class Mode { kAdaptUp, kAdaptDown } mode_;
+ };
+
+ void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(encoder_queue_);
+ DegradationPreference EffectiveDegradataionPreference()
+ RTC_RUN_ON(encoder_queue_);
+ AdaptCounter& GetAdaptCounter() RTC_RUN_ON(encoder_queue_);
+ bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const
+ RTC_RUN_ON(encoder_queue_);
+
+ rtc::TaskQueue* encoder_queue_;
+ DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_);
+ // Counters used for deciding if the video resolution or framerate is
+ // currently restricted, and if so, why, on a per degradation preference
+ // basis.
+ // TODO(sprang): Replace this with a state holding a relative overuse measure
+ // instead, that can be translated into suitable down-scale or fps limit.
+ std::map<const DegradationPreference, AdaptCounter> adapt_counters_
+ RTC_GUARDED_BY(encoder_queue_);
+ const BalancedDegradationSettings balanced_settings_
+ RTC_GUARDED_BY(encoder_queue_);
+ // Stores a snapshot of the last adaptation request triggered by an AdaptUp
+ // or AdaptDown signal.
+ absl::optional<AdaptationRequest> last_adaptation_request_
+ RTC_GUARDED_BY(encoder_queue_);
+ absl::optional<int> last_frame_pixel_count_ RTC_GUARDED_BY(encoder_queue_);
+ // The source proxy may modify its source or sink off the |encoder_queue_|.
+ const std::unique_ptr<VideoSourceProxy> source_proxy_;
+ const std::unique_ptr<OveruseFrameDetector> overuse_detector_
+ RTC_PT_GUARDED_BY(encoder_queue_);
+ int codec_max_framerate_ RTC_GUARDED_BY(encoder_queue_);
+ uint32_t encoder_start_bitrate_bps_ RTC_GUARDED_BY(encoder_queue_);
+ bool is_quality_scaler_enabled_ RTC_GUARDED_BY(encoder_queue_);
+ VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(encoder_queue_);
+ VideoEncoder* encoder_ RTC_GUARDED_BY(encoder_queue_);
+ VideoStreamEncoderObserver* const encoder_stats_observer_
+ RTC_GUARDED_BY(encoder_queue_);
+};
+
+} // namespace webrtc
+
+#endif // VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 446b1b6..ccf00de 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -45,7 +45,6 @@
// Time interval for logging frame counts.
const int64_t kFrameLogIntervalMs = 60000;
-const int kMinFramerateFps = 2;
// Time to keep a single cached pending frame in paused state.
const int64_t kPendingFrameTimeoutMs = 1000;
@@ -214,268 +213,42 @@
}
} // namespace
-// VideoSourceProxy is responsible ensuring thread safety between calls to
-// VideoStreamEncoder::SetSource that will happen on libjingle's worker thread
-// when a video capturer is connected to the encoder and the encoder task queue
-// (encoder_queue_) where the encoder reports its VideoSinkWants.
-class VideoStreamEncoder::VideoSourceProxy {
- public:
- explicit VideoSourceProxy(VideoStreamEncoder* video_stream_encoder)
- : video_stream_encoder_(video_stream_encoder),
- degradation_preference_(DegradationPreference::DISABLED),
- source_(nullptr),
- max_framerate_(std::numeric_limits<int>::max()),
- max_pixels_(std::numeric_limits<int>::max()),
- resolution_alignment_(1) {}
+absl::optional<VideoEncoder::ResolutionBitrateLimits> GetEncoderBitrateLimits(
+ const VideoEncoder::EncoderInfo& encoder_info,
+ int frame_size_pixels) {
+ std::vector<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
+ encoder_info.resolution_bitrate_limits;
- void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
- const DegradationPreference& degradation_preference) {
- // Called on libjingle's worker thread.
- RTC_DCHECK_RUN_ON(&main_checker_);
- rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr;
- rtc::VideoSinkWants wants;
- {
- rtc::CritScope lock(&crit_);
- degradation_preference_ = degradation_preference;
- old_source = source_;
- source_ = source;
- wants = GetActiveSinkWantsInternal();
+ // Sort the list of bitrate limits by resolution.
+ sort(bitrate_limits.begin(), bitrate_limits.end(),
+ [](const VideoEncoder::ResolutionBitrateLimits& lhs,
+ const VideoEncoder::ResolutionBitrateLimits& rhs) {
+ return lhs.frame_size_pixels < rhs.frame_size_pixels;
+ });
+
+ for (size_t i = 0; i < bitrate_limits.size(); ++i) {
+ RTC_DCHECK_GT(bitrate_limits[i].min_bitrate_bps, 0);
+ RTC_DCHECK_GT(bitrate_limits[i].min_start_bitrate_bps, 0);
+ RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
+ bitrate_limits[i].min_bitrate_bps);
+ if (i > 0) {
+ // The bitrate limits aren't expected to decrease with resolution.
+ RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps,
+ bitrate_limits[i - 1].min_bitrate_bps);
+ RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps,
+ bitrate_limits[i - 1].min_start_bitrate_bps);
+ RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
+ bitrate_limits[i - 1].max_bitrate_bps);
}
- if (old_source != source && old_source != nullptr) {
- old_source->RemoveSink(video_stream_encoder_);
- }
-
- if (!source) {
- return;
- }
-
- source->AddOrUpdateSink(video_stream_encoder_, wants);
- }
-
- void SetMaxFramerateAndAlignment(int max_framerate,
- int resolution_alignment) {
- RTC_DCHECK_GT(max_framerate, 0);
- rtc::CritScope lock(&crit_);
- if (max_framerate == max_framerate_ &&
- resolution_alignment == resolution_alignment_) {
- return;
- }
-
- RTC_LOG(LS_INFO) << "Set max framerate: " << max_framerate
- << " and resolution alignment: " << resolution_alignment;
- max_framerate_ = max_framerate;
- resolution_alignment_ = resolution_alignment;
- if (source_) {
- source_->AddOrUpdateSink(video_stream_encoder_,
- GetActiveSinkWantsInternal());
+ if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) {
+ return absl::optional<VideoEncoder::ResolutionBitrateLimits>(
+ bitrate_limits[i]);
}
}
- void SetWantsRotationApplied(bool rotation_applied) {
- rtc::CritScope lock(&crit_);
- sink_wants_.rotation_applied = rotation_applied;
- if (source_) {
- source_->AddOrUpdateSink(video_stream_encoder_,
- GetActiveSinkWantsInternal());
- }
- }
-
- rtc::VideoSinkWants GetActiveSinkWants() {
- rtc::CritScope lock(&crit_);
- return GetActiveSinkWantsInternal();
- }
-
- void ResetPixelFpsCount() {
- rtc::CritScope lock(&crit_);
- sink_wants_.max_pixel_count = std::numeric_limits<int>::max();
- sink_wants_.target_pixel_count.reset();
- sink_wants_.max_framerate_fps = std::numeric_limits<int>::max();
- if (source_)
- source_->AddOrUpdateSink(video_stream_encoder_,
- GetActiveSinkWantsInternal());
- }
-
- bool RequestResolutionLowerThan(int pixel_count,
- int min_pixels_per_frame,
- bool* min_pixels_reached) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
- // This can happen since |degradation_preference_| is set on libjingle's
- // worker thread but the adaptation is done on the encoder task queue.
- return false;
- }
- // The input video frame size will have a resolution less than or equal to
- // |max_pixel_count| depending on how the source can scale the frame size.
- const int pixels_wanted = (pixel_count * 3) / 5;
- if (pixels_wanted >= sink_wants_.max_pixel_count) {
- return false;
- }
- if (pixels_wanted < min_pixels_per_frame) {
- *min_pixels_reached = true;
- return false;
- }
- RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: "
- << pixels_wanted;
- sink_wants_.max_pixel_count = pixels_wanted;
- sink_wants_.target_pixel_count = absl::nullopt;
- source_->AddOrUpdateSink(video_stream_encoder_,
- GetActiveSinkWantsInternal());
- return true;
- }
-
- int RequestFramerateLowerThan(int fps) {
- // Called on the encoder task queue.
- // The input video frame rate will be scaled down to 2/3, rounding down.
- int framerate_wanted = (fps * 2) / 3;
- return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
- }
-
- int GetHigherResolutionThan(int pixel_count) const {
- // On step down we request at most 3/5 the pixel count of the previous
- // resolution, so in order to take "one step up" we request a resolution
- // as close as possible to 5/3 of the current resolution. The actual pixel
- // count selected depends on the capabilities of the source. In order to
- // not take a too large step up, we cap the requested pixel count to be at
- // most four time the current number of pixels.
- return (pixel_count * 5) / 3;
- }
-
- bool RequestHigherResolutionThan(int pixel_count) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
- // This can happen since |degradation_preference_| is set on libjingle's
- // worker thread but the adaptation is done on the encoder task queue.
- return false;
- }
- int max_pixels_wanted = pixel_count;
- if (max_pixels_wanted != std::numeric_limits<int>::max())
- max_pixels_wanted = pixel_count * 4;
-
- if (max_pixels_wanted <= sink_wants_.max_pixel_count)
- return false;
-
- sink_wants_.max_pixel_count = max_pixels_wanted;
- if (max_pixels_wanted == std::numeric_limits<int>::max()) {
- // Remove any constraints.
- sink_wants_.target_pixel_count.reset();
- } else {
- sink_wants_.target_pixel_count = GetHigherResolutionThan(pixel_count);
- }
- RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
- << max_pixels_wanted;
- source_->AddOrUpdateSink(video_stream_encoder_,
- GetActiveSinkWantsInternal());
- return true;
- }
-
- // Request upgrade in framerate. Returns the new requested frame, or -1 if
- // no change requested. Note that maxint may be returned if limits due to
- // adaptation requests are removed completely. In that case, consider
- // |max_framerate_| to be the current limit (assuming the capturer complies).
- int RequestHigherFramerateThan(int fps) {
- // Called on the encoder task queue.
- // The input frame rate will be scaled up to the last step, with rounding.
- int framerate_wanted = fps;
- if (fps != std::numeric_limits<int>::max())
- framerate_wanted = (fps * 3) / 2;
-
- return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1;
- }
-
- bool RestrictFramerate(int fps) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
- return false;
-
- const int fps_wanted = std::max(kMinFramerateFps, fps);
- if (fps_wanted >= sink_wants_.max_framerate_fps)
- return false;
-
- RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted;
- sink_wants_.max_framerate_fps = fps_wanted;
- source_->AddOrUpdateSink(video_stream_encoder_,
- GetActiveSinkWantsInternal());
- return true;
- }
-
- bool IncreaseFramerate(int fps) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
- return false;
-
- const int fps_wanted = std::max(kMinFramerateFps, fps);
- if (fps_wanted <= sink_wants_.max_framerate_fps)
- return false;
-
- RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted;
- sink_wants_.max_framerate_fps = fps_wanted;
- source_->AddOrUpdateSink(video_stream_encoder_,
- GetActiveSinkWantsInternal());
- return true;
- }
-
- // Used in automatic animation detection for screenshare.
- bool RestrictPixels(int max_pixels) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
- // This can happen since |degradation_preference_| is set on libjingle's
- // worker thread but the adaptation is done on the encoder task queue.
- return false;
- }
- max_pixels_ = max_pixels;
- RTC_LOG(LS_INFO) << "Applying max pixel restriction: " << max_pixels;
- source_->AddOrUpdateSink(video_stream_encoder_,
- GetActiveSinkWantsInternal());
- return true;
- }
-
- private:
- rtc::VideoSinkWants GetActiveSinkWantsInternal()
- RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
- rtc::VideoSinkWants wants = sink_wants_;
- // Clear any constraints from the current sink wants that don't apply to
- // the used degradation_preference.
- switch (degradation_preference_) {
- case DegradationPreference::BALANCED:
- break;
- case DegradationPreference::MAINTAIN_FRAMERATE:
- wants.max_framerate_fps = std::numeric_limits<int>::max();
- break;
- case DegradationPreference::MAINTAIN_RESOLUTION:
- wants.max_pixel_count = std::numeric_limits<int>::max();
- wants.target_pixel_count.reset();
- break;
- case DegradationPreference::DISABLED:
- wants.max_pixel_count = std::numeric_limits<int>::max();
- wants.target_pixel_count.reset();
- wants.max_framerate_fps = std::numeric_limits<int>::max();
- }
- // Limit to configured max framerate.
- wants.max_framerate_fps = std::min(max_framerate_, wants.max_framerate_fps);
- // Limit resolution due to automatic animation detection for screenshare.
- wants.max_pixel_count = std::min(max_pixels_, wants.max_pixel_count);
- wants.resolution_alignment = resolution_alignment_;
- return wants;
- }
-
- rtc::CriticalSection crit_;
- SequenceChecker main_checker_;
- VideoStreamEncoder* const video_stream_encoder_;
- rtc::VideoSinkWants sink_wants_ RTC_GUARDED_BY(&crit_);
- DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_);
- rtc::VideoSourceInterface<VideoFrame>* source_ RTC_GUARDED_BY(&crit_);
- int max_framerate_ RTC_GUARDED_BY(&crit_);
- int max_pixels_ RTC_GUARDED_BY(&crit_);
- int resolution_alignment_ RTC_GUARDED_BY(&crit_);
-
- RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
-};
+ return absl::nullopt;
+}
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
: rate_control(),
@@ -519,12 +292,10 @@
quality_rampup_done_(false),
quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()),
quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
- source_proxy_(new VideoSourceProxy(this)),
sink_(nullptr),
settings_(settings),
rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()),
- overuse_detector_(std::move(overuse_detector)),
encoder_stats_observer_(encoder_stats_observer),
encoder_initialized_(false),
max_framerate_(-1),
@@ -541,7 +312,6 @@
was_encode_called_since_last_initialization_(false),
encoder_failed_(false),
clock_(clock),
- degradation_preference_(DegradationPreference::DISABLED),
posted_frames_waiting_for_encode_(0),
last_captured_timestamp_(0),
delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
@@ -568,12 +338,17 @@
automatic_animation_detection_experiment_(
ParseAutomatincAnimationDetectionFieldTrial()),
encoder_switch_requested_(false),
+ resource_adaptation_module_(
+ std::make_unique<OveruseFrameDetectorResourceAdaptationModule>(
+ this,
+ std::move(overuse_detector),
+ encoder_stats_observer)),
encoder_queue_(task_queue_factory->CreateTaskQueue(
"EncoderQueue",
TaskQueueFactory::Priority::NORMAL)) {
RTC_DCHECK(encoder_stats_observer);
- RTC_DCHECK(overuse_detector_);
RTC_DCHECK_GE(number_of_cores, 1);
+ resource_adaptation_module_->Initialize(encoder_queue());
for (auto& state : encoder_buffer_state_)
state.fill(std::numeric_limits<int64_t>::max());
@@ -587,14 +362,15 @@
void VideoStreamEncoder::Stop() {
RTC_DCHECK_RUN_ON(&thread_checker_);
- source_proxy_->SetSource(nullptr, DegradationPreference());
+ resource_adaptation_module_->SetSource(nullptr, DegradationPreference());
encoder_queue_.PostTask([this] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- overuse_detector_->StopCheckForOveruse();
+ resource_adaptation_module_->StopCheckForOveruse();
rate_allocator_ = nullptr;
bitrate_observer_ = nullptr;
ReleaseEncoder();
quality_scaler_ = nullptr;
+ resource_adaptation_module_->SetIsQualityScalerEnabled(false);
shutdown_event_.Set();
});
@@ -627,23 +403,9 @@
rtc::VideoSourceInterface<VideoFrame>* source,
const DegradationPreference& degradation_preference) {
RTC_DCHECK_RUN_ON(&thread_checker_);
- source_proxy_->SetSource(source, degradation_preference);
+ resource_adaptation_module_->SetSource(source, degradation_preference);
encoder_queue_.PostTask([this, degradation_preference] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- if (degradation_preference_ != degradation_preference) {
- // Reset adaptation state, so that we're not tricked into thinking there's
- // an already pending request of the same type.
- last_adaptation_request_.reset();
- if (degradation_preference == DegradationPreference::BALANCED ||
- degradation_preference_ == DegradationPreference::BALANCED) {
- // TODO(asapersson): Consider removing |adapt_counters_| map and use one
- // AdaptCounter for all modes.
- source_proxy_->ResetPixelFpsCount();
- adapt_counters_.clear();
- }
- }
- degradation_preference_ = degradation_preference;
-
if (encoder_)
ConfigureQualityScaler(encoder_->GetEncoderInfo());
@@ -651,13 +413,13 @@
max_framerate_ != -1) {
// If frame rate scaling is no longer allowed, remove any potential
// allowance for longer frame intervals.
- overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
+ resource_adaptation_module_->RefreshTargetFramerate();
}
});
}
void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
- source_proxy_->SetWantsRotationApplied(rotation_applied);
+ resource_adaptation_module_->SetSourceWantsRotationApplied(rotation_applied);
encoder_queue_.PostTask([this, sink] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
sink_ = sink;
@@ -668,6 +430,8 @@
encoder_queue_.PostTask([this, start_bitrate_bps] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
encoder_start_bitrate_bps_ = start_bitrate_bps;
+ resource_adaptation_module_->SetEncoderStartBitrateBps(
+ encoder_start_bitrate_bps_);
set_start_bitrate_bps_ = start_bitrate_bps;
set_start_bitrate_time_ms_ = clock_->TimeInMilliseconds();
});
@@ -685,6 +449,7 @@
(!encoder_ || encoder_config_.video_format != config.video_format ||
max_data_payload_length_ != max_data_payload_length);
encoder_config_ = std::move(config);
+ resource_adaptation_module_->SetEncoderConfig(encoder_config_.Copy());
max_data_payload_length_ = max_data_payload_length;
pending_encoder_reconfiguration_ = true;
@@ -700,49 +465,14 @@
encoder_config_.video_format);
if (HasInternalSource()) {
last_frame_info_ = VideoFrameInfo(176, 144, false);
+ resource_adaptation_module_->SetLastFramePixelCount(
+ last_frame_info_->pixel_count());
ReconfigureEncoder();
}
}
});
}
-static absl::optional<VideoEncoder::ResolutionBitrateLimits>
-GetEncoderBitrateLimits(const VideoEncoder::EncoderInfo& encoder_info,
- int frame_size_pixels) {
- std::vector<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
- encoder_info.resolution_bitrate_limits;
-
- // Sort the list of bitrate limits by resolution.
- sort(bitrate_limits.begin(), bitrate_limits.end(),
- [](const VideoEncoder::ResolutionBitrateLimits& lhs,
- const VideoEncoder::ResolutionBitrateLimits& rhs) {
- return lhs.frame_size_pixels < rhs.frame_size_pixels;
- });
-
- for (size_t i = 0; i < bitrate_limits.size(); ++i) {
- RTC_DCHECK_GT(bitrate_limits[i].min_bitrate_bps, 0);
- RTC_DCHECK_GT(bitrate_limits[i].min_start_bitrate_bps, 0);
- RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
- bitrate_limits[i].min_bitrate_bps);
- if (i > 0) {
- // The bitrate limits aren't expected to decrease with resolution.
- RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps,
- bitrate_limits[i - 1].min_bitrate_bps);
- RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps,
- bitrate_limits[i - 1].min_start_bitrate_bps);
- RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
- bitrate_limits[i - 1].max_bitrate_bps);
- }
-
- if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) {
- return absl::optional<VideoEncoder::ResolutionBitrateLimits>(
- bitrate_limits[i]);
- }
- }
-
- return absl::nullopt;
-}
-
// TODO(bugs.webrtc.org/8807): Currently this always does a hard
// reconfiguration, but this isn't always necessary. Add in logic to only update
// the VideoBitrateAllocator and call OnEncoderConfigurationChanged with a
@@ -793,6 +523,7 @@
encoder_ = settings_.encoder_factory->CreateVideoEncoder(
encoder_config_.video_format);
+ resource_adaptation_module_->SetEncoder(encoder_.get());
// TODO(nisse): What to do if creating the encoder fails? Crash,
// or just discard incoming frames?
RTC_CHECK(encoder_);
@@ -882,13 +613,14 @@
// Make sure the start bit rate is sane...
RTC_DCHECK_LE(codec.startBitrate, 1000000);
max_framerate_ = codec.maxFramerate;
+ resource_adaptation_module_->SetCodecMaxFramerate(max_framerate_);
// Inform source about max configured framerate.
int max_framerate = 0;
for (const auto& stream : streams) {
max_framerate = std::max(stream.max_framerate, max_framerate);
}
- source_proxy_->SetMaxFramerateAndAlignment(
+ resource_adaptation_module_->SetSourceMaxFramerateAndAlignment(
max_framerate, encoder_->GetEncoderInfo().requested_resolution_alignment);
if (codec.maxBitrate == 0) {
@@ -970,12 +702,9 @@
}
if (pending_encoder_creation_) {
- overuse_detector_->StopCheckForOveruse();
- overuse_detector_->StartCheckForOveruse(
- &encoder_queue_,
- GetCpuOveruseOptions(
- settings_, encoder_->GetEncoderInfo().is_hardware_accelerated),
- this);
+ resource_adaptation_module_->StopCheckForOveruse();
+ resource_adaptation_module_->StartCheckForOveruse(GetCpuOveruseOptions(
+ settings_, encoder_->GetEncoderInfo().is_hardware_accelerated));
pending_encoder_creation_ = false;
}
@@ -1030,14 +759,7 @@
std::move(streams), encoder_config_.content_type,
encoder_config_.min_transmit_bitrate_bps);
- // Get the current target framerate, ie the maximum framerate as specified by
- // the current codec configuration, or any limit imposed by cpu adaption in
- // maintain-resolution or balanced mode. This is used to make sure overuse
- // detection doesn't needlessly trigger in low and/or variable framerate
- // scenarios.
- int target_framerate = std::min(
- max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps);
- overuse_detector_->OnTargetFramerateUpdated(target_framerate);
+ resource_adaptation_module_->RefreshTargetFramerate();
ConfigureQualityScaler(info);
}
@@ -1047,7 +769,8 @@
RTC_DCHECK_RUN_ON(&encoder_queue_);
const auto scaling_settings = encoder_info.scaling_settings;
const bool quality_scaling_allowed =
- IsResolutionScalingEnabled(degradation_preference_) &&
+ IsResolutionScalingEnabled(
+ resource_adaptation_module_->degradation_preference()) &&
scaling_settings.thresholds;
if (quality_scaling_allowed) {
@@ -1062,24 +785,26 @@
}
// Since the interface is non-public, std::make_unique can't do this
// upcast.
- AdaptationObserverInterface* observer = this;
+ AdaptationObserverInterface* observer = resource_adaptation_module_.get();
quality_scaler_ = std::make_unique<QualityScaler>(
&encoder_queue_, observer,
experimental_thresholds ? *experimental_thresholds
: *(scaling_settings.thresholds));
+ resource_adaptation_module_->SetIsQualityScalerEnabled(true);
has_seen_first_significant_bwe_change_ = false;
initial_framedrop_ = 0;
}
} else {
quality_scaler_.reset(nullptr);
+ resource_adaptation_module_->SetIsQualityScalerEnabled(false);
initial_framedrop_ = kMaxInitialFramedrop;
}
- if (degradation_preference_ == DegradationPreference::BALANCED &&
+ if (resource_adaptation_module_->degradation_preference() ==
+ DegradationPreference::BALANCED &&
quality_scaler_ && last_frame_info_) {
absl::optional<VideoEncoder::QpThresholds> thresholds =
- balanced_settings_.GetQpThresholds(encoder_config_.codec_type,
- last_frame_info_->pixel_count());
+ resource_adaptation_module_->GetQpThresholds();
if (thresholds) {
quality_scaler_->SetQpThresholds(*thresholds);
}
@@ -1087,7 +812,10 @@
encoder_stats_observer_->OnAdaptationChanged(
VideoStreamEncoderObserver::AdaptationReason::kNone,
- GetActiveCounts(kCpu), GetActiveCounts(kQuality));
+ resource_adaptation_module_->GetActiveCounts(
+ AdaptationObserverInterface::AdaptReason::kCpu),
+ resource_adaptation_module_->GetActiveCounts(
+ AdaptationObserverInterface::AdaptReason::kQuality));
}
void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) {
@@ -1322,6 +1050,8 @@
pending_encoder_reconfiguration_ = true;
last_frame_info_ = VideoFrameInfo(video_frame.width(), video_frame.height(),
video_frame.is_texture());
+ resource_adaptation_module_->SetLastFramePixelCount(
+ last_frame_info_->pixel_count());
RTC_LOG(LS_INFO) << "Video frame parameters changed: dimensions="
<< last_frame_info_->width << "x"
<< last_frame_info_->height
@@ -1376,15 +1106,22 @@
if (DropDueToSize(video_frame.size())) {
RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
- int fps_count = GetConstAdaptCounter().FramerateCount(kQuality);
- int res_count = GetConstAdaptCounter().ResolutionCount(kQuality);
- AdaptDown(kQuality);
- if (degradation_preference_ == DegradationPreference::BALANCED &&
- GetConstAdaptCounter().FramerateCount(kQuality) > fps_count) {
+ int fps_count =
+ resource_adaptation_module_->GetConstAdaptCounter().FramerateCount(
+ AdaptationObserverInterface::AdaptReason::kQuality);
+ int res_count =
+ resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount(
+ AdaptationObserverInterface::AdaptReason::kQuality);
+ TriggerAdaptDown(AdaptationObserverInterface::AdaptReason::kQuality);
+ if (resource_adaptation_module_->degradation_preference() ==
+ DegradationPreference::BALANCED &&
+ resource_adaptation_module_->GetConstAdaptCounter().FramerateCount(
+ AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) {
// Adapt framerate in same step as resolution.
- AdaptDown(kQuality);
+ TriggerAdaptDown(AdaptationObserverInterface::AdaptReason::kQuality);
}
- if (GetConstAdaptCounter().ResolutionCount(kQuality) > res_count) {
+ if (resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount(
+ AdaptationObserverInterface::AdaptReason::kQuality) > res_count) {
encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
}
++initial_framedrop_;
@@ -1404,12 +1141,12 @@
initial_framedrop_ = kMaxInitialFramedrop;
if (!quality_rampup_done_ && TryQualityRampup(now_ms) &&
- GetConstAdaptCounter().ResolutionCount(kQuality) > 0 &&
- GetConstAdaptCounter().TotalCount(kCpu) == 0) {
+ resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount(
+ AdaptationObserverInterface::AdaptReason::kQuality) > 0 &&
+ resource_adaptation_module_->GetConstAdaptCounter().TotalCount(
+ AdaptationObserverInterface::AdaptReason::kCpu) == 0) {
RTC_LOG(LS_INFO) << "Reset quality limitations.";
- last_adaptation_request_.reset();
- source_proxy_->ResetPixelFpsCount();
- adapt_counters_.clear();
+ resource_adaptation_module_->ResetAdaptationCounters();
quality_rampup_done_ = true;
}
@@ -1593,7 +1330,7 @@
TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
"Encode");
- overuse_detector_->FrameCaptured(out_frame, time_when_posted_us);
+ resource_adaptation_module_->FrameCaptured(out_frame, time_when_posted_us);
RTC_DCHECK_LE(send_codec_.width, out_frame.width());
RTC_DCHECK_LE(send_codec_.height, out_frame.height());
@@ -1922,6 +1659,8 @@
encoder_start_bitrate_bps_ = target_bitrate.bps() != 0
? target_bitrate.bps()
: encoder_start_bitrate_bps_;
+ resource_adaptation_module_->SetEncoderStartBitrateBps(
+ encoder_start_bitrate_bps_);
if (video_suspension_changed) {
RTC_LOG(LS_INFO) << "Video suspend state changed to: "
@@ -1979,277 +1718,14 @@
return false;
}
-bool VideoStreamEncoder::AdaptDown(AdaptReason reason) {
- RTC_DCHECK_RUN_ON(&encoder_queue_);
- AdaptationRequest adaptation_request = {
- last_frame_info_->pixel_count(),
- encoder_stats_observer_->GetInputFrameRate(),
- AdaptationRequest::Mode::kAdaptDown};
-
- bool downgrade_requested =
- last_adaptation_request_ &&
- last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
-
- bool did_adapt = true;
-
- switch (EffectiveDegradataionPreference()) {
- case DegradationPreference::BALANCED:
- break;
- case DegradationPreference::MAINTAIN_FRAMERATE:
- if (downgrade_requested &&
- adaptation_request.input_pixel_count_ >=
- last_adaptation_request_->input_pixel_count_) {
- // Don't request lower resolution if the current resolution is not
- // lower than the last time we asked for the resolution to be lowered.
- return true;
- }
- break;
- case DegradationPreference::MAINTAIN_RESOLUTION:
- if (adaptation_request.framerate_fps_ <= 0 ||
- (downgrade_requested &&
- adaptation_request.framerate_fps_ < kMinFramerateFps)) {
- // If no input fps estimate available, can't determine how to scale down
- // framerate. Otherwise, don't request lower framerate if we don't have
- // a valid frame rate. Since framerate, unlike resolution, is a measure
- // we have to estimate, and can fluctuate naturally over time, don't
- // make the same kind of limitations as for resolution, but trust the
- // overuse detector to not trigger too often.
- return true;
- }
- break;
- case DegradationPreference::DISABLED:
- return true;
- }
-
- switch (EffectiveDegradataionPreference()) {
- case DegradationPreference::BALANCED: {
- // Try scale down framerate, if lower.
- int fps = balanced_settings_.MinFps(encoder_config_.codec_type,
- last_frame_info_->pixel_count());
- if (source_proxy_->RestrictFramerate(fps)) {
- GetAdaptCounter().IncrementFramerate(reason);
- // Check if requested fps is higher (or close to) input fps.
- absl::optional<int> min_diff =
- balanced_settings_.MinFpsDiff(last_frame_info_->pixel_count());
- if (min_diff && adaptation_request.framerate_fps_ > 0) {
- int fps_diff = adaptation_request.framerate_fps_ - fps;
- if (fps_diff < min_diff.value()) {
- did_adapt = false;
- }
- }
- break;
- }
- // Scale down resolution.
- RTC_FALLTHROUGH();
- }
- case DegradationPreference::MAINTAIN_FRAMERATE: {
- // Scale down resolution.
- bool min_pixels_reached = false;
- if (!source_proxy_->RequestResolutionLowerThan(
- adaptation_request.input_pixel_count_,
- encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame,
- &min_pixels_reached)) {
- if (min_pixels_reached)
- encoder_stats_observer_->OnMinPixelLimitReached();
- return true;
- }
- GetAdaptCounter().IncrementResolution(reason);
- break;
- }
- case DegradationPreference::MAINTAIN_RESOLUTION: {
- // Scale down framerate.
- const int requested_framerate = source_proxy_->RequestFramerateLowerThan(
- adaptation_request.framerate_fps_);
- if (requested_framerate == -1)
- return true;
- RTC_DCHECK_NE(max_framerate_, -1);
- overuse_detector_->OnTargetFramerateUpdated(
- std::min(max_framerate_, requested_framerate));
- GetAdaptCounter().IncrementFramerate(reason);
- break;
- }
- case DegradationPreference::DISABLED:
- RTC_NOTREACHED();
- }
-
- last_adaptation_request_.emplace(adaptation_request);
-
- UpdateAdaptationStats(reason);
-
- RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString();
- return did_adapt;
+bool VideoStreamEncoder::TriggerAdaptDown(
+ AdaptationObserverInterface::AdaptReason reason) {
+ return resource_adaptation_module_->AdaptDown(reason);
}
-void VideoStreamEncoder::AdaptUp(AdaptReason reason) {
- RTC_DCHECK_RUN_ON(&encoder_queue_);
-
- const AdaptCounter& adapt_counter = GetConstAdaptCounter();
- int num_downgrades = adapt_counter.TotalCount(reason);
- if (num_downgrades == 0)
- return;
- RTC_DCHECK_GT(num_downgrades, 0);
-
- AdaptationRequest adaptation_request = {
- last_frame_info_->pixel_count(),
- encoder_stats_observer_->GetInputFrameRate(),
- AdaptationRequest::Mode::kAdaptUp};
-
- bool adapt_up_requested =
- last_adaptation_request_ &&
- last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
-
- if (EffectiveDegradataionPreference() ==
- DegradationPreference::MAINTAIN_FRAMERATE) {
- if (adapt_up_requested &&
- adaptation_request.input_pixel_count_ <=
- last_adaptation_request_->input_pixel_count_) {
- // Don't request higher resolution if the current resolution is not
- // higher than the last time we asked for the resolution to be higher.
- return;
- }
- }
-
- switch (EffectiveDegradataionPreference()) {
- case DegradationPreference::BALANCED: {
- // Check if quality should be increased based on bitrate.
- if (reason == kQuality &&
- !balanced_settings_.CanAdaptUp(last_frame_info_->pixel_count(),
- encoder_start_bitrate_bps_)) {
- return;
- }
- // Try scale up framerate, if higher.
- int fps = balanced_settings_.MaxFps(encoder_config_.codec_type,
- last_frame_info_->pixel_count());
- if (source_proxy_->IncreaseFramerate(fps)) {
- GetAdaptCounter().DecrementFramerate(reason, fps);
- // Reset framerate in case of fewer fps steps down than up.
- if (adapt_counter.FramerateCount() == 0 &&
- fps != std::numeric_limits<int>::max()) {
- RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
- source_proxy_->IncreaseFramerate(std::numeric_limits<int>::max());
- }
- break;
- }
- // Check if resolution should be increased based on bitrate.
- if (reason == kQuality &&
- !balanced_settings_.CanAdaptUpResolution(
- last_frame_info_->pixel_count(), encoder_start_bitrate_bps_)) {
- return;
- }
- // Scale up resolution.
- RTC_FALLTHROUGH();
- }
- case DegradationPreference::MAINTAIN_FRAMERATE: {
- // Check if resolution should be increased based on bitrate and
- // limits specified by encoder capabilities.
- if (reason == kQuality &&
- !CanAdaptUpResolution(last_frame_info_->pixel_count(),
- encoder_start_bitrate_bps_)) {
- return;
- }
-
- // Scale up resolution.
- int pixel_count = adaptation_request.input_pixel_count_;
- if (adapt_counter.ResolutionCount() == 1) {
- RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
- pixel_count = std::numeric_limits<int>::max();
- }
- if (!source_proxy_->RequestHigherResolutionThan(pixel_count))
- return;
- GetAdaptCounter().DecrementResolution(reason);
- break;
- }
- case DegradationPreference::MAINTAIN_RESOLUTION: {
- // Scale up framerate.
- int fps = adaptation_request.framerate_fps_;
- if (adapt_counter.FramerateCount() == 1) {
- RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
- fps = std::numeric_limits<int>::max();
- }
-
- const int requested_framerate =
- source_proxy_->RequestHigherFramerateThan(fps);
- if (requested_framerate == -1) {
- overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
- return;
- }
- overuse_detector_->OnTargetFramerateUpdated(
- std::min(max_framerate_, requested_framerate));
- GetAdaptCounter().DecrementFramerate(reason);
- break;
- }
- case DegradationPreference::DISABLED:
- return;
- }
-
- last_adaptation_request_.emplace(adaptation_request);
-
- UpdateAdaptationStats(reason);
-
- RTC_LOG(LS_INFO) << adapt_counter.ToString();
-}
-
-bool VideoStreamEncoder::CanAdaptUpResolution(int pixels,
- uint32_t bitrate_bps) const {
- absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
- GetEncoderBitrateLimits(encoder_info_,
- source_proxy_->GetHigherResolutionThan(pixels));
- if (!bitrate_limits.has_value() || bitrate_bps == 0) {
- return true; // No limit configured or bitrate provided.
- }
- RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels);
- return bitrate_bps >=
- static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
-}
-
-// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
-void VideoStreamEncoder::UpdateAdaptationStats(AdaptReason reason) {
- switch (reason) {
- case kCpu:
- encoder_stats_observer_->OnAdaptationChanged(
- VideoStreamEncoderObserver::AdaptationReason::kCpu,
- GetActiveCounts(kCpu), GetActiveCounts(kQuality));
- break;
- case kQuality:
- encoder_stats_observer_->OnAdaptationChanged(
- VideoStreamEncoderObserver::AdaptationReason::kQuality,
- GetActiveCounts(kCpu), GetActiveCounts(kQuality));
- break;
- }
-}
-
-VideoStreamEncoderObserver::AdaptationSteps VideoStreamEncoder::GetActiveCounts(
- AdaptReason reason) {
- VideoStreamEncoderObserver::AdaptationSteps counts =
- GetConstAdaptCounter().Counts(reason);
- switch (reason) {
- case kCpu:
- if (!IsFramerateScalingEnabled(degradation_preference_))
- counts.num_framerate_reductions = absl::nullopt;
- if (!IsResolutionScalingEnabled(degradation_preference_))
- counts.num_resolution_reductions = absl::nullopt;
- break;
- case kQuality:
- if (!IsFramerateScalingEnabled(degradation_preference_) ||
- !quality_scaler_) {
- counts.num_framerate_reductions = absl::nullopt;
- }
- if (!IsResolutionScalingEnabled(degradation_preference_) ||
- !quality_scaler_) {
- counts.num_resolution_reductions = absl::nullopt;
- }
- break;
- }
- return counts;
-}
-
-VideoStreamEncoder::AdaptCounter& VideoStreamEncoder::GetAdaptCounter() {
- return adapt_counters_[degradation_preference_];
-}
-
-const VideoStreamEncoder::AdaptCounter&
-VideoStreamEncoder::GetConstAdaptCounter() {
- return adapt_counters_[degradation_preference_];
+void VideoStreamEncoder::TriggerAdaptUp(
+ AdaptationObserverInterface::AdaptReason reason) {
+ resource_adaptation_module_->AdaptUp(reason);
}
void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image,
@@ -2294,7 +1770,7 @@
}
}
- overuse_detector_->FrameSent(
+ resource_adaptation_module_->FrameSent(
encoded_image.Timestamp(), time_sent_us,
encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec,
encode_duration_us);
@@ -2320,116 +1796,6 @@
TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release");
}
-// Class holding adaptation information.
-VideoStreamEncoder::AdaptCounter::AdaptCounter() {
- fps_counters_.resize(kScaleReasonSize);
- resolution_counters_.resize(kScaleReasonSize);
- static_assert(kScaleReasonSize == 2, "Update MoveCount.");
-}
-
-VideoStreamEncoder::AdaptCounter::~AdaptCounter() {}
-
-std::string VideoStreamEncoder::AdaptCounter::ToString() const {
- rtc::StringBuilder ss;
- ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
- ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
- return ss.Release();
-}
-
-VideoStreamEncoderObserver::AdaptationSteps
-VideoStreamEncoder::AdaptCounter::Counts(int reason) const {
- VideoStreamEncoderObserver::AdaptationSteps counts;
- counts.num_framerate_reductions = fps_counters_[reason];
- counts.num_resolution_reductions = resolution_counters_[reason];
- return counts;
-}
-
-void VideoStreamEncoder::AdaptCounter::IncrementFramerate(int reason) {
- ++(fps_counters_[reason]);
-}
-
-void VideoStreamEncoder::AdaptCounter::IncrementResolution(int reason) {
- ++(resolution_counters_[reason]);
-}
-
-void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason) {
- if (fps_counters_[reason] == 0) {
- // Balanced mode: Adapt up is in a different order, switch reason.
- // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3).
- // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0}
- // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0}
- // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0}
- // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0}
- RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
- RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
- MoveCount(&resolution_counters_, reason);
- MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize);
- }
- --(fps_counters_[reason]);
- RTC_DCHECK_GE(fps_counters_[reason], 0);
-}
-
-void VideoStreamEncoder::AdaptCounter::DecrementResolution(int reason) {
- if (resolution_counters_[reason] == 0) {
- // Balanced mode: Adapt up is in a different order, switch reason.
- RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
- RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
- MoveCount(&fps_counters_, reason);
- MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize);
- }
- --(resolution_counters_[reason]);
- RTC_DCHECK_GE(resolution_counters_[reason], 0);
-}
-
-void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason,
- int cur_fps) {
- DecrementFramerate(reason);
- // Reset if at max fps (i.e. in case of fewer steps up than down).
- if (cur_fps == std::numeric_limits<int>::max())
- absl::c_fill(fps_counters_, 0);
-}
-
-int VideoStreamEncoder::AdaptCounter::FramerateCount() const {
- return Count(fps_counters_);
-}
-
-int VideoStreamEncoder::AdaptCounter::ResolutionCount() const {
- return Count(resolution_counters_);
-}
-
-int VideoStreamEncoder::AdaptCounter::FramerateCount(int reason) const {
- return fps_counters_[reason];
-}
-
-int VideoStreamEncoder::AdaptCounter::ResolutionCount(int reason) const {
- return resolution_counters_[reason];
-}
-
-int VideoStreamEncoder::AdaptCounter::TotalCount(int reason) const {
- return FramerateCount(reason) + ResolutionCount(reason);
-}
-
-int VideoStreamEncoder::AdaptCounter::Count(
- const std::vector<int>& counters) const {
- return absl::c_accumulate(counters, 0);
-}
-
-void VideoStreamEncoder::AdaptCounter::MoveCount(std::vector<int>* counters,
- int from_reason) {
- int to_reason = (from_reason + 1) % kScaleReasonSize;
- ++((*counters)[to_reason]);
- --((*counters)[from_reason]);
-}
-
-std::string VideoStreamEncoder::AdaptCounter::ToString(
- const std::vector<int>& counters) const {
- rtc::StringBuilder ss;
- for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
- ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
- }
- return ss.Release();
-}
-
bool VideoStreamEncoder::EncoderSwitchExperiment::IsBitrateBelowThreshold(
const DataRate& target_bitrate) {
DataRate rate =
@@ -2562,7 +1928,8 @@
if (!automatic_animation_detection_experiment_.enabled ||
encoder_config_.content_type !=
VideoEncoderConfig::ContentType::kScreen ||
- degradation_preference_ != DegradationPreference::BALANCED) {
+ resource_adaptation_module_->degradation_preference() !=
+ DegradationPreference::BALANCED) {
return;
}
@@ -2619,23 +1986,10 @@
RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent "
"animation detection.";
}
- source_proxy_->RestrictPixels(should_cap_resolution
- ? kMaxAnimationPixels
- : std::numeric_limits<int>::max());
+ resource_adaptation_module_->SetSourceMaxPixels(
+ should_cap_resolution ? kMaxAnimationPixels
+ : std::numeric_limits<int>::max());
}
}
-DegradationPreference VideoStreamEncoder::EffectiveDegradataionPreference()
- const {
- // Balanced mode for screenshare works via automatic animation detection:
- // Resolution is capped for fullscreen animated content.
- // Adapatation is done only via framerate downgrade.
- // Thus effective degradation preference is MAINTAIN_RESOLUTION.
- return (encoder_config_.content_type ==
- VideoEncoderConfig::ContentType::kScreen &&
- degradation_preference_ == DegradationPreference::BALANCED)
- ? DegradationPreference::MAINTAIN_RESOLUTION
- : degradation_preference_;
-}
-
} // namespace webrtc
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 9517944..f180688 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -30,7 +30,6 @@
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/event.h"
-#include "rtc_base/experiments/balanced_degradation_settings.h"
#include "rtc_base/experiments/quality_rampup_experiment.h"
#include "rtc_base/experiments/quality_scaler_settings.h"
#include "rtc_base/experiments/rate_control_settings.h"
@@ -42,10 +41,14 @@
#include "system_wrappers/include/clock.h"
#include "video/encoder_bitrate_adjuster.h"
#include "video/frame_encode_metadata_writer.h"
-#include "video/overuse_frame_detector.h"
+#include "video/overuse_frame_detector_resource_adaptation_module.h"
namespace webrtc {
+absl::optional<VideoEncoder::ResolutionBitrateLimits> GetEncoderBitrateLimits(
+ const VideoEncoder::EncoderInfo& encoder_info,
+ int frame_size_pixels);
+
// VideoStreamEncoder represent a video encoder that accepts raw video frames as
// input and produces an encoded bit stream.
// Usage:
@@ -55,9 +58,7 @@
// Call ConfigureEncoder with the codec settings.
// Call Stop() when done.
class VideoStreamEncoder : public VideoStreamEncoderInterface,
- private EncodedImageCallback,
- // Protected only to provide access to tests.
- protected AdaptationObserverInterface {
+ private EncodedImageCallback {
public:
VideoStreamEncoder(Clock* clock,
uint32_t number_of_cores,
@@ -104,14 +105,14 @@
// be called on |encoder_queue_|.
rtc::TaskQueue* encoder_queue() { return &encoder_queue_; }
- // AdaptationObserverInterface implementation.
// These methods are protected for easier testing.
- void AdaptUp(AdaptReason reason) override;
- bool AdaptDown(AdaptReason reason) override;
+ // TODO(hbos): When "DropDueToSize" no longer causes TriggerAdaptDown(), these
+ // methods are only used for testing and can be removed in favor of the test
+ // invoking AdaptUp() or AdaptDown() on a test-injected adaptation module.
+ void TriggerAdaptUp(AdaptationObserverInterface::AdaptReason reason);
+ bool TriggerAdaptDown(AdaptationObserverInterface::AdaptReason reason);
private:
- class VideoSourceProxy;
-
class VideoFrameInfo {
public:
VideoFrameInfo(int width, int height, bool is_texture)
@@ -183,50 +184,6 @@
void SetEncoderRates(const EncoderRateSettings& rate_settings)
RTC_RUN_ON(&encoder_queue_);
- // Class holding adaptation information.
- class AdaptCounter final {
- public:
- AdaptCounter();
- ~AdaptCounter();
-
- // Get number of adaptation downscales for |reason|.
- VideoStreamEncoderObserver::AdaptationSteps Counts(int reason) const;
-
- std::string ToString() const;
-
- void IncrementFramerate(int reason);
- void IncrementResolution(int reason);
- void DecrementFramerate(int reason);
- void DecrementResolution(int reason);
- void DecrementFramerate(int reason, int cur_fps);
-
- // Gets the total number of downgrades (for all adapt reasons).
- int FramerateCount() const;
- int ResolutionCount() const;
-
- // Gets the total number of downgrades for |reason|.
- int FramerateCount(int reason) const;
- int ResolutionCount(int reason) const;
- int TotalCount(int reason) const;
-
- private:
- std::string ToString(const std::vector<int>& counters) const;
- int Count(const std::vector<int>& counters) const;
- void MoveCount(std::vector<int>* counters, int from_reason);
-
- // Degradation counters holding number of framerate/resolution reductions
- // per adapt reason.
- std::vector<int> fps_counters_;
- std::vector<int> resolution_counters_;
- };
-
- AdaptCounter& GetAdaptCounter() RTC_RUN_ON(&encoder_queue_);
- const AdaptCounter& GetConstAdaptCounter() RTC_RUN_ON(&encoder_queue_);
- void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(&encoder_queue_);
- VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
- AdaptReason reason) RTC_RUN_ON(&encoder_queue_);
- bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const
- RTC_RUN_ON(&encoder_queue_);
void RunPostEncode(EncodedImage encoded_image,
int64_t time_sent_us,
int temporal_index,
@@ -238,10 +195,6 @@
int64_t time_when_posted_in_ms)
RTC_RUN_ON(&encoder_queue_);
- // Calculates degradation preference used in adaptation down or up.
- DegradationPreference EffectiveDegradataionPreference() const
- RTC_RUN_ON(&encoder_queue_);
-
rtc::Event shutdown_event_;
const uint32_t number_of_cores_;
@@ -255,14 +208,11 @@
const bool quality_scaling_experiment_enabled_;
- const std::unique_ptr<VideoSourceProxy> source_proxy_;
EncoderSink* sink_;
const VideoStreamEncoderSettings settings_;
const RateControlSettings rate_control_settings_;
const QualityScalerSettings quality_scaler_settings_;
- const std::unique_ptr<OveruseFrameDetector> overuse_detector_
- RTC_PT_GUARDED_BY(&encoder_queue_);
std::unique_ptr<QualityScaler> quality_scaler_ RTC_GUARDED_BY(&encoder_queue_)
RTC_PT_GUARDED_BY(&encoder_queue_);
@@ -277,8 +227,6 @@
bool encoder_initialized_;
std::unique_ptr<VideoBitrateAllocator> rate_allocator_
RTC_GUARDED_BY(&encoder_queue_) RTC_PT_GUARDED_BY(&encoder_queue_);
- // The maximum frame rate of the current codec configuration, as determined
- // at the last ReconfigureEncoder() call.
int max_framerate_ RTC_GUARDED_BY(&encoder_queue_);
// Set when ConfigureEncoder has been called in order to lazy reconfigure the
@@ -308,30 +256,6 @@
bool encoder_failed_ RTC_GUARDED_BY(&encoder_queue_);
Clock* const clock_;
- // Counters used for deciding if the video resolution or framerate is
- // currently restricted, and if so, why, on a per degradation preference
- // basis.
- // TODO(sprang): Replace this with a state holding a relative overuse measure
- // instead, that can be translated into suitable down-scale or fps limit.
- std::map<const DegradationPreference, AdaptCounter> adapt_counters_
- RTC_GUARDED_BY(&encoder_queue_);
- // Set depending on degradation preferences.
- DegradationPreference degradation_preference_ RTC_GUARDED_BY(&encoder_queue_);
-
- const BalancedDegradationSettings balanced_settings_;
-
- struct AdaptationRequest {
- // The pixel count produced by the source at the time of the adaptation.
- int input_pixel_count_;
- // Framerate received from the source at the time of the adaptation.
- int framerate_fps_;
- // Indicates if request was to adapt up or down.
- enum class Mode { kAdaptUp, kAdaptDown } mode_;
- };
- // Stores a snapshot of the last adaptation request triggered by an AdaptUp
- // or AdaptDown signal.
- absl::optional<AdaptationRequest> last_adaptation_request_
- RTC_GUARDED_BY(&encoder_queue_);
rtc::RaceChecker incoming_frame_race_checker_
RTC_GUARDED_BY(incoming_frame_race_checker_);
@@ -473,6 +397,9 @@
// track of whether a request has been made or not.
bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_);
+ std::unique_ptr<OveruseFrameDetectorResourceAdaptationModule>
+ resource_adaptation_module_;
+
// All public methods are proxied to |encoder_queue_|. It must must be
// destroyed first to make sure no tasks are run that use other members.
rtc::TaskQueue encoder_queue_;
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 57b2b35..4b4cd2d 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -155,17 +155,20 @@
new CpuOveruseDetectorProxy(stats_proxy)),
task_queue_factory) {}
- void PostTaskAndWait(bool down, AdaptReason reason) {
+ void PostTaskAndWait(bool down,
+ AdaptationObserverInterface::AdaptReason reason) {
PostTaskAndWait(down, reason, /*expected_results=*/true);
}
- void PostTaskAndWait(bool down, AdaptReason reason, bool expected_results) {
+ void PostTaskAndWait(bool down,
+ AdaptationObserverInterface::AdaptReason reason,
+ bool expected_results) {
rtc::Event event;
encoder_queue()->PostTask([this, &event, reason, down, expected_results] {
if (down)
- EXPECT_EQ(expected_results, AdaptDown(reason));
+ EXPECT_EQ(expected_results, TriggerAdaptDown(reason));
else
- AdaptUp(reason);
+ TriggerAdaptUp(reason);
event.Set();
});
ASSERT_TRUE(event.Wait(5000));
@@ -180,24 +183,29 @@
}
void TriggerCpuOveruse() {
- PostTaskAndWait(/*down=*/true, AdaptReason::kCpu);
+ PostTaskAndWait(/*down=*/true,
+ AdaptationObserverInterface::AdaptReason::kCpu);
}
void TriggerCpuNormalUsage() {
- PostTaskAndWait(/*down=*/false, AdaptReason::kCpu);
+ PostTaskAndWait(/*down=*/false,
+ AdaptationObserverInterface::AdaptReason::kCpu);
}
void TriggerQualityLow() {
- PostTaskAndWait(/*down=*/true, AdaptReason::kQuality);
+ PostTaskAndWait(/*down=*/true,
+ AdaptationObserverInterface::AdaptReason::kQuality);
}
void TriggerQualityLowExpectFalse() {
- PostTaskAndWait(/*down=*/true, AdaptReason::kQuality,
+ PostTaskAndWait(/*down=*/true,
+ AdaptationObserverInterface::AdaptReason::kQuality,
/*expected_results=*/false);
}
void TriggerQualityHigh() {
- PostTaskAndWait(/*down=*/false, AdaptReason::kQuality);
+ PostTaskAndWait(/*down=*/false,
+ AdaptationObserverInterface::AdaptReason::kQuality);
}
CpuOveruseDetectorProxy* overuse_detector_proxy_;
