[Overuse] Add EncodeUsageResource and QualityScalerResource.
This refactors the usage of OveruseFrameDetector in
OveruseFrameDetectorResourceAdaptationModule into an inner class of the
module, making the interaction between the detector and the module the
responsibility of this helper class instead.
Similarly, QualityScaler usage is moved into QualityScalerResource.
This takes us one step closer to separate the act of detecting
overuse/underuse of a resource and the logic of what to do when
overuse/underuse happens.
Follow-up CLs should build on this in order to materialize the concept
of having resources, streams and a central decision-maker deciding how
to reconfigure the streams based on resource usage state.
Bug: webrtc:11222
Change-Id: I99a08a42218a871db8f477f31447a6379433ad05
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168057
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30468}
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
index edced6d..5671607 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.cc
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -72,12 +72,154 @@
} // namespace
+// Handles interaction with the OveruseDetector.
+class OveruseFrameDetectorResourceAdaptationModule::EncodeUsageResource
+ : public AdaptationObserverInterface {
+ public:
+ EncodeUsageResource(OveruseFrameDetectorResourceAdaptationModule* module,
+ std::unique_ptr<OveruseFrameDetector> overuse_detector)
+ : module_(module),
+ overuse_detector_(std::move(overuse_detector)),
+ is_started_(false),
+ target_frame_rate_(absl::nullopt) {
+ RTC_DCHECK(module_);
+ RTC_DCHECK(overuse_detector_);
+ }
+
+ void StartCheckForOveruse(CpuOveruseOptions options) {
+ RTC_DCHECK(!is_started_);
+ overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(),
+ std::move(options), this);
+ is_started_ = true;
+ overuse_detector_->OnTargetFramerateUpdated(TargetFrameRateAsInt());
+ }
+
+ void StopCheckForOveruse() {
+ overuse_detector_->StopCheckForOveruse();
+ is_started_ = false;
+ }
+
+ void SetTargetFrameRate(absl::optional<double> target_frame_rate) {
+ if (target_frame_rate == target_frame_rate_)
+ return;
+ target_frame_rate_ = target_frame_rate;
+ if (is_started_)
+ overuse_detector_->OnTargetFramerateUpdated(TargetFrameRateAsInt());
+ }
+
+ void OnEncodeStarted(const VideoFrame& cropped_frame,
+ int64_t time_when_first_seen_us) {
+ // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g.
+ // "OnEncodeStarted"?) or revise usage.
+ overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us);
+ }
+
+ void OnEncodeCompleted(uint32_t timestamp,
+ int64_t time_sent_in_us,
+ int64_t capture_time_us,
+ absl::optional<int> encode_duration_us) {
+ // TODO(hbos): Rename FrameSent() to something more appropriate (e.g.
+ // "OnEncodeCompleted"?).
+ overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us,
+ encode_duration_us);
+ }
+
+ // AdaptationObserverInterface implementation.
+ void AdaptUp(AdaptReason reason) override {
+ RTC_DCHECK_EQ(reason, AdaptReason::kCpu);
+ module_->OnResourceUnderuse(reason);
+ }
+ bool AdaptDown(AdaptReason reason) override {
+ RTC_DCHECK_EQ(reason, AdaptReason::kCpu);
+ return module_->OnResourceOveruse(reason);
+ }
+
+ private:
+ int TargetFrameRateAsInt() {
+ return target_frame_rate_.has_value()
+ ? static_cast<int>(target_frame_rate_.value())
+ : std::numeric_limits<int>::max();
+ }
+
+ OveruseFrameDetectorResourceAdaptationModule* const module_;
+ const std::unique_ptr<OveruseFrameDetector> overuse_detector_;
+ bool is_started_;
+ absl::optional<double> target_frame_rate_;
+};
+
+// Handles interaction with the QualityScaler.
+class OveruseFrameDetectorResourceAdaptationModule::QualityScalerResource
+ : public AdaptationObserverInterface {
+ public:
+ explicit QualityScalerResource(
+ OveruseFrameDetectorResourceAdaptationModule* module)
+ : module_(module), quality_scaler_(nullptr) {
+ RTC_DCHECK(module_);
+ }
+
+ bool is_started() const { return quality_scaler_.get(); }
+ // TODO(https://crbug.com/webrtc/11222): Don't expose the quality scaler.
+ QualityScaler* quality_scaler() const { return quality_scaler_.get(); }
+
+ void StartCheckForOveruse(VideoEncoder::QpThresholds qp_thresholds) {
+ RTC_DCHECK(!is_started());
+ quality_scaler_ =
+ std::make_unique<QualityScaler>(this, std::move(qp_thresholds));
+ }
+
+ void StopCheckForOveruse() { quality_scaler_.reset(); }
+
+ void SetQpThresholds(VideoEncoder::QpThresholds qp_thresholds) {
+ RTC_DCHECK(is_started());
+ quality_scaler_->SetQpThresholds(std::move(qp_thresholds));
+ }
+
+ bool QpFastFilterLow() {
+ RTC_DCHECK(is_started());
+ return quality_scaler_->QpFastFilterLow();
+ }
+
+ void OnEncodeCompleted(const EncodedImage& encoded_image,
+ int64_t time_sent_in_us) {
+ if (quality_scaler_ && encoded_image.qp_ >= 0)
+ quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us);
+ }
+
+ void OnFrameDropped(EncodedImageCallback::DropReason reason) {
+ if (!quality_scaler_)
+ return;
+ switch (reason) {
+ case EncodedImageCallback::DropReason::kDroppedByMediaOptimizations:
+ quality_scaler_->ReportDroppedFrameByMediaOpt();
+ break;
+ case EncodedImageCallback::DropReason::kDroppedByEncoder:
+ quality_scaler_->ReportDroppedFrameByEncoder();
+ break;
+ }
+ }
+
+ // AdaptationObserverInterface implementation.
+ void AdaptUp(AdaptReason reason) override {
+ RTC_DCHECK_EQ(reason, AdaptReason::kQuality);
+ module_->OnResourceUnderuse(reason);
+ }
+ bool AdaptDown(AdaptReason reason) override {
+ RTC_DCHECK_EQ(reason, AdaptReason::kQuality);
+ return module_->OnResourceOveruse(reason);
+ }
+
+ private:
+ OveruseFrameDetectorResourceAdaptationModule* const module_;
+ std::unique_ptr<QualityScaler> quality_scaler_;
+};
+
// VideoSourceRestrictor is responsible for keeping track of current
// VideoSourceRestrictions and how to modify them in response to adapting up or
// down. It is not reponsible for determining when we should adapt up or down -
-// for that, see OveruseFrameDetectorResourceAdaptationModule::AdaptUp() and
-// AdaptDown() - only how to modify the source/sink restrictions when this
-// happens. Note that it is also not responsible for reconfigruring the
+// for that, see
+// OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse() and
+// OnResourceOveruse() - only how to modify the source/sink restrictions when
+// this happens. Note that it is also not responsible for reconfigruring the
// source/sink, it is only a keeper of desired restrictions.
class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
public:
@@ -224,9 +366,10 @@
class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final {
public:
AdaptCounter() {
- fps_counters_.resize(kScaleReasonSize);
- resolution_counters_.resize(kScaleReasonSize);
- static_assert(kScaleReasonSize == 2, "Update MoveCount.");
+ fps_counters_.resize(AdaptationObserverInterface::kScaleReasonSize);
+ resolution_counters_.resize(AdaptationObserverInterface::kScaleReasonSize);
+ static_assert(AdaptationObserverInterface::kScaleReasonSize == 2,
+ "Update MoveCount.");
}
~AdaptCounter() = default;
@@ -258,7 +401,8 @@
RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
MoveCount(&resolution_counters_, reason);
- MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize);
+ MoveCount(&fps_counters_,
+ (reason + 1) % AdaptationObserverInterface::kScaleReasonSize);
}
--(fps_counters_[reason]);
RTC_DCHECK_GE(fps_counters_[reason], 0);
@@ -270,7 +414,8 @@
RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
MoveCount(&fps_counters_, reason);
- MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize);
+ MoveCount(&resolution_counters_,
+ (reason + 1) % AdaptationObserverInterface::kScaleReasonSize);
}
--(resolution_counters_[reason]);
RTC_DCHECK_GE(resolution_counters_[reason], 0);
@@ -297,7 +442,8 @@
private:
std::string ToString(const std::vector<int>& counters) const {
rtc::StringBuilder ss;
- for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
+ for (size_t reason = 0;
+ reason < AdaptationObserverInterface::kScaleReasonSize; ++reason) {
ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
}
return ss.Release();
@@ -308,7 +454,8 @@
}
void MoveCount(std::vector<int>* counters, int from_reason) {
- int to_reason = (from_reason + 1) % kScaleReasonSize;
+ int to_reason =
+ (from_reason + 1) % AdaptationObserverInterface::kScaleReasonSize;
++((*counters)[to_reason]);
--((*counters)[from_reason]);
}
@@ -335,13 +482,14 @@
balanced_settings_(),
last_adaptation_request_(absl::nullopt),
source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
- overuse_detector_(std::move(overuse_detector)),
- overuse_detector_is_started_(false),
+ encode_usage_resource_(
+ std::make_unique<EncodeUsageResource>(this,
+ std::move(overuse_detector))),
+ quality_scaler_resource_(std::make_unique<QualityScalerResource>(this)),
+ quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
last_input_frame_size_(absl::nullopt),
target_frame_rate_(absl::nullopt),
encoder_target_bitrate_bps_(absl::nullopt),
- quality_scaler_(nullptr),
- quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()),
quality_rampup_done_(false),
quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()),
@@ -349,7 +497,6 @@
encoder_stats_observer_(encoder_stats_observer),
initial_framedrop_(0) {
RTC_DCHECK(adaptation_listener_);
- RTC_DCHECK(overuse_detector_);
RTC_DCHECK(encoder_stats_observer_);
}
@@ -359,32 +506,21 @@
void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation(
ResourceAdaptationModuleListener* adaptation_listener) {
RTC_DCHECK(encoder_settings_.has_value());
- RTC_DCHECK(!overuse_detector_is_started_);
- // TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside
- // the interval between StartCheckForOveruse() and StopCheckForOveruse(),
- // support configuring which |adaptation_listener_| to use on the fly. It is
- // currently hardcoded for the entire lifetime of the module in order to
- // support adaptation caused by VideoStreamEncoder or QualityScaler invoking
- // AdaptUp() and AdaptDown() even when the OveruseDetector is inactive.
+ // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener
+ // should be passed in and why. If resources are separated from modules then
+ // those resources may be started or stopped separately from the module.
RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_);
- overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(),
- GetCpuOveruseOptions(), this);
- overuse_detector_is_started_ = true;
- overuse_detector_->OnTargetFramerateUpdated(
- target_frame_rate_.has_value()
- ? static_cast<int>(target_frame_rate_.value())
- : std::numeric_limits<int>::max());
+ encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions());
}
void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() {
- overuse_detector_->StopCheckForOveruse();
- overuse_detector_is_started_ = false;
- quality_scaler_.reset();
+ encode_usage_resource_->StopCheckForOveruse();
+ quality_scaler_resource_->StopCheckForOveruse();
}
void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo(
bool has_input_video) {
- // While false, AdaptUp() and AdaptDown() are NO-OPS.
+ // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS.
has_input_video_ = has_input_video;
}
@@ -430,8 +566,11 @@
encoder_target_bitrate_bps_ = target_bitrate.bps();
// Check for bwe drop experiment
+ // TODO(https://crbug.com/webrtc/11222): Should this move to
+ // QualityScalerResource?
if (start_bitrate_.set_start_bitrate_ > DataRate::Zero() &&
- !start_bitrate_.has_seen_first_bwe_drop_ && quality_scaler_ &&
+ !start_bitrate_.has_seen_first_bwe_drop_ &&
+ quality_scaler_resource_->is_started() &&
quality_scaler_settings_.InitialBitrateIntervalMs() &&
quality_scaler_settings_.InitialBitrateFactor()) {
int64_t diff_ms = clock_->TimeInMilliseconds() -
@@ -472,12 +611,12 @@
AdaptationObserverInterface::AdaptReason::kQuality);
int res_count = GetConstAdaptCounter().ResolutionCount(
AdaptationObserverInterface::AdaptReason::kQuality);
- AdaptDown(AdaptationObserverInterface::AdaptReason::kQuality);
+ OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
if (degradation_preference() == DegradationPreference::BALANCED &&
GetConstAdaptCounter().FramerateCount(
AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) {
// Adapt framerate in same step as resolution.
- AdaptDown(AdaptationObserverInterface::AdaptReason::kQuality);
+ OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
}
if (GetConstAdaptCounter().ResolutionCount(
AdaptationObserverInterface::AdaptReason::kQuality) > res_count) {
@@ -489,39 +628,27 @@
void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted(
const VideoFrame& cropped_frame,
int64_t time_when_first_seen_us) {
- // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g.
- // "OnEncodeStarted"?) or revise usage.
- overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us);
+ encode_usage_resource_->OnEncodeStarted(cropped_frame,
+ time_when_first_seen_us);
}
void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted(
const EncodedImage& encoded_image,
int64_t time_sent_in_us,
absl::optional<int> encode_duration_us) {
- // TODO(hbos): Rename FrameSent() to something more appropriate (e.g.
- // "OnEncodeCompleted"?).
+ // Inform |encode_usage_resource_| of the encode completed event.
uint32_t timestamp = encoded_image.Timestamp();
int64_t capture_time_us =
encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
- overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us,
- encode_duration_us);
- if (quality_scaler_ && encoded_image.qp_ >= 0)
- quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us);
+ encode_usage_resource_->OnEncodeCompleted(
+ timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
+ // Inform |quality_scaler_resource_| of the encode completed event.
+ quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
}
void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped(
EncodedImageCallback::DropReason reason) {
- if (!quality_scaler_) {
- return;
- }
- switch (reason) {
- case EncodedImageCallback::DropReason::kDroppedByMediaOptimizations:
- quality_scaler_->ReportDroppedFrameByMediaOpt();
- break;
- case EncodedImageCallback::DropReason::kDroppedByEncoder:
- quality_scaler_->ReportDroppedFrameByEncoder();
- break;
- }
+ quality_scaler_resource_->OnFrameDropped(reason);
}
void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() {
@@ -536,12 +663,12 @@
void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
if (qp_thresholds.has_value()) {
- quality_scaler_ =
- std::make_unique<QualityScaler>(this, qp_thresholds.value());
+ quality_scaler_resource_->StopCheckForOveruse();
+ quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value());
// Restart frame drops due to size.
initial_framedrop_ = 0;
} else {
- quality_scaler_ = nullptr;
+ quality_scaler_resource_->StopCheckForOveruse();
// Quality scaling disabled so we shouldn't drop initial frames.
initial_framedrop_ = kMaxInitialFramedrop;
}
@@ -554,8 +681,10 @@
IsResolutionScalingEnabled(degradation_preference_) &&
scaling_settings.thresholds;
+ // TODO(https://crbug.com/webrtc/11222): Should this move to
+ // QualityScalerResource?
if (quality_scaling_allowed) {
- if (quality_scaler_ == nullptr) {
+ if (!quality_scaler_resource_->is_started()) {
// Quality scaler has not already been configured.
// Use experimental thresholds if available.
@@ -574,12 +703,12 @@
// Set the qp-thresholds to the balanced settings if balanced mode.
if (degradation_preference_ == DegradationPreference::BALANCED &&
- quality_scaler_) {
+ quality_scaler_resource_->is_started()) {
absl::optional<VideoEncoder::QpThresholds> thresholds =
balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
LastInputFrameSizeOrDefault());
if (thresholds) {
- quality_scaler_->SetQpThresholds(*thresholds);
+ quality_scaler_resource_->SetQpThresholds(*thresholds);
}
}
@@ -589,7 +718,8 @@
GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality));
}
-void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
+void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse(
+ AdaptationObserverInterface::AdaptReason reason) {
if (!has_input_video_)
return;
const AdaptCounter& adapt_counter = GetConstAdaptCounter();
@@ -621,7 +751,7 @@
switch (EffectiveDegradataionPreference()) {
case DegradationPreference::BALANCED: {
// Check if quality should be increased based on bitrate.
- if (reason == kQuality &&
+ if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
!balanced_settings_.CanAdaptUp(
GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault(),
encoder_target_bitrate_bps_.value_or(0))) {
@@ -642,7 +772,7 @@
break;
}
// Check if resolution should be increased based on bitrate.
- if (reason == kQuality &&
+ if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
!balanced_settings_.CanAdaptUpResolution(
GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault(),
encoder_target_bitrate_bps_.value_or(0))) {
@@ -654,7 +784,7 @@
case DegradationPreference::MAINTAIN_FRAMERATE: {
// Check if resolution should be increased based on bitrate and
// limits specified by encoder capabilities.
- if (reason == kQuality &&
+ if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
!CanAdaptUpResolution(LastInputFrameSizeOrDefault(),
encoder_target_bitrate_bps_.value_or(0))) {
return;
@@ -702,8 +832,8 @@
RTC_LOG(LS_INFO) << adapt_counter.ToString();
}
-bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
- AdaptReason reason) {
+bool OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse(
+ AdaptationObserverInterface::AdaptReason reason) {
if (!has_input_video_)
return false;
AdaptationRequest adaptation_request = {
@@ -847,8 +977,8 @@
// yet.
// TODO(hbos): Can we simply DCHECK has_value() before usage instead? Having a
// DCHECK passed all the tests but adding it does change the requirements of
- // this class (= not being allowed to call AdaptUp() or AdaptDown() before
- // OnFrame()) and deserves a standalone CL.
+ // this class (= not being allowed to call OnResourceUnderuse() or
+ // OnResourceOveruse() before OnFrame()) and deserves a standalone CL.
return last_input_frame_size_.value_or(
VideoStreamEncoder::kDefaultLastFrameInfoWidth *
VideoStreamEncoder::kDefaultLastFrameInfoHeight);
@@ -886,53 +1016,47 @@
codec_max_frame_rate.value() < target_frame_rate.value())) {
target_frame_rate = codec_max_frame_rate;
}
- if (target_frame_rate != target_frame_rate_) {
- target_frame_rate_ = target_frame_rate;
- if (overuse_detector_is_started_) {
- overuse_detector_->OnTargetFramerateUpdated(
- target_frame_rate_.has_value()
- ? static_cast<int>(target_frame_rate_.value())
- : std::numeric_limits<int>::max());
- }
- }
+ encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
}
// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats(
- AdaptReason reason) {
+ AdaptationObserverInterface::AdaptReason reason) {
switch (reason) {
- case kCpu:
+ case AdaptationObserverInterface::AdaptReason::kCpu:
encoder_stats_observer_->OnAdaptationChanged(
VideoStreamEncoderObserver::AdaptationReason::kCpu,
- GetActiveCounts(kCpu), GetActiveCounts(kQuality));
+ GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu),
+ GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality));
break;
- case kQuality:
+ case AdaptationObserverInterface::AdaptReason::kQuality:
encoder_stats_observer_->OnAdaptationChanged(
VideoStreamEncoderObserver::AdaptationReason::kQuality,
- GetActiveCounts(kCpu), GetActiveCounts(kQuality));
+ GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu),
+ GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality));
break;
}
}
VideoStreamEncoderObserver::AdaptationSteps
OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
- AdaptReason reason) {
+ AdaptationObserverInterface::AdaptReason reason) {
VideoStreamEncoderObserver::AdaptationSteps counts =
GetConstAdaptCounter().Counts(reason);
switch (reason) {
- case kCpu:
+ case AdaptationObserverInterface::AdaptReason::kCpu:
if (!IsFramerateScalingEnabled(degradation_preference_))
counts.num_framerate_reductions = absl::nullopt;
if (!IsResolutionScalingEnabled(degradation_preference_))
counts.num_resolution_reductions = absl::nullopt;
break;
- case kQuality:
+ case AdaptationObserverInterface::AdaptReason::kQuality:
if (!IsFramerateScalingEnabled(degradation_preference_) ||
- !quality_scaler_) {
+ !quality_scaler_resource_->is_started()) {
counts.num_framerate_reductions = absl::nullopt;
}
if (!IsResolutionScalingEnabled(degradation_preference_) ||
- !quality_scaler_) {
+ !quality_scaler_resource_->is_started()) {
counts.num_resolution_reductions = absl::nullopt;
}
break;
@@ -983,7 +1107,7 @@
void OveruseFrameDetectorResourceAdaptationModule::
MaybePerformQualityRampupExperiment() {
- if (!quality_scaler_)
+ if (!quality_scaler_resource_->is_started())
return;
if (quality_rampup_done_)
@@ -1000,7 +1124,7 @@
if (encoder_settings_ &&
encoder_target_bitrate_bps_.value_or(0) ==
encoder_settings_->video_codec().maxBitrate * 1000 &&
- quality_scaler_->QpFastFilterLow()) {
+ quality_scaler_resource_->QpFastFilterLow()) {
try_quality_rampup = true;
}
}
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
index 6c841dc..437510b 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.h
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -51,8 +51,7 @@
// generic interface in VideoStreamEncoder, unblocking other modules from being
// implemented and used.
class OveruseFrameDetectorResourceAdaptationModule
- : public ResourceAdaptationModuleInterface,
- public AdaptationObserverInterface {
+ : public ResourceAdaptationModuleInterface {
public:
// The module can be constructed on any sequence, but must be initialized and
// used on a single sequence, e.g. the encoder queue.
@@ -91,6 +90,11 @@
absl::optional<int> encode_duration_us) override;
void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
+ // TODO(hbos): Is dropping initial frames really just a special case of "don't
+ // encode frames right now"? Can this be part of VideoSourceRestrictions,
+ // which handles the output of the rest of the encoder settings? This is
+ // something we'll need to support for "disable video due to overuse", not
+ // initial frames.
bool DropInitialFrames() const;
// TODO(eshr): This can be made private if we configure on
@@ -98,20 +102,20 @@
// (https://crbug.com/webrtc/11338)
void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
- // AdaptationObserverInterface implementation. Used both "internally" as
- // feedback from |overuse_detector_|, and externally from VideoStreamEncoder:
- // - It is wired to the VideoStreamEncoder::quality_scaler_.
- // - It is invoked by VideoStreamEncoder::MaybeEncodeVideoFrame().
- // TODO(hbos): Decouple quality scaling and resource adaptation, or find an
- // interface for reconfiguring externally.
- // TODO(hbos): VideoStreamEncoder should not be responsible for any part of
- // the adaptation.
- void AdaptUp(AdaptReason reason) override;
- bool AdaptDown(AdaptReason reason) override;
+ // Signal that a resource (kCpu or kQuality) is overused or underused. This is
+ // currently used by EncodeUsageResource, QualityScalerResource and testing.
+ // TODO(https://crbug.com/webrtc/11222): Make use of ResourceUsageState and
+ // implement resources per call/adaptation/resource.h. When adaptation happens
+ // because a resource is in specific usage state, get rid of these explicit
+ // triggers.
+ void OnResourceUnderuse(AdaptationObserverInterface::AdaptReason reason);
+ bool OnResourceOveruse(AdaptationObserverInterface::AdaptReason reason);
private:
- class AdaptCounter;
+ class EncodeUsageResource;
+ class QualityScalerResource;
class VideoSourceRestrictor;
+ class AdaptCounter;
struct AdaptationRequest {
// The pixel count produced by the source at the time of the adaptation.
@@ -132,23 +136,22 @@
VideoCodecType GetVideoCodecTypeOrGeneric() const;
int LastInputFrameSizeOrDefault() const;
VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
- AdaptReason reason);
+ AdaptationObserverInterface::AdaptReason reason);
const AdaptCounter& GetConstAdaptCounter();
// Makes |video_source_restrictions_| up-to-date and informs the
// |adaptation_listener_| if restrictions are changed, allowing the listener
// to reconfigure the source accordingly.
void MaybeUpdateVideoSourceRestrictions();
- // Calculates an up-to-date value of |target_frame_rate_| and informs the
- // |overuse_detector_| of the new value if it changed and the detector is
- // started.
+ // Calculates an up-to-date value of the target frame rate and informs the
+ // |encode_usage_resource_| of the new value.
void MaybeUpdateTargetFrameRate();
// Use nullopt to disable quality scaling.
void UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
- void UpdateAdaptationStats(AdaptReason reason);
+ void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason);
DegradationPreference EffectiveDegradataionPreference();
AdaptCounter& GetAdaptCounter();
bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const;
@@ -180,15 +183,14 @@
absl::optional<AdaptationRequest> last_adaptation_request_;
// Keeps track of source restrictions that this adaptation module outputs.
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
- const std::unique_ptr<OveruseFrameDetector> overuse_detector_;
- bool overuse_detector_is_started_;
+ const std::unique_ptr<EncodeUsageResource> encode_usage_resource_;
+ const std::unique_ptr<QualityScalerResource> quality_scaler_resource_;
+ const bool quality_scaling_experiment_enabled_;
absl::optional<int> last_input_frame_size_;
absl::optional<double> target_frame_rate_;
// This is the last non-zero target bitrate for the encoder.
absl::optional<uint32_t> encoder_target_bitrate_bps_;
absl::optional<VideoEncoder::RateControlParameters> encoder_rates_;
- std::unique_ptr<QualityScaler> quality_scaler_;
- const bool quality_scaling_experiment_enabled_;
const QualityScalerSettings quality_scaler_settings_;
bool quality_rampup_done_;
QualityRampupExperiment quality_rampup_experiment_;
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index e90dd0c..a8fbac9 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -1577,16 +1577,16 @@
return false;
}
-bool VideoStreamEncoder::TriggerAdaptDown(
+void VideoStreamEncoder::OnResourceUnderuseForTesting(
AdaptationObserverInterface::AdaptReason reason) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- return resource_adaptation_module_->AdaptDown(reason);
+ resource_adaptation_module_->OnResourceUnderuse(reason);
}
-void VideoStreamEncoder::TriggerAdaptUp(
+bool VideoStreamEncoder::OnResourceOveruseForTesting(
AdaptationObserverInterface::AdaptReason reason) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- resource_adaptation_module_->AdaptUp(reason);
+ return resource_adaptation_module_->OnResourceOveruse(reason);
}
void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 465d611..fee1067 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -117,12 +117,13 @@
// be called on |encoder_queue_|.
rtc::TaskQueue* encoder_queue() { return &encoder_queue_; }
- // These methods are protected for easier testing.
- // TODO(hbos): When "DropDueToSize" no longer causes TriggerAdaptDown(), these
- // methods are only used for testing and can be removed in favor of the test
- // invoking AdaptUp() or AdaptDown() on a test-injected adaptation module.
- void TriggerAdaptUp(AdaptationObserverInterface::AdaptReason reason);
- bool TriggerAdaptDown(AdaptationObserverInterface::AdaptReason reason);
+ // TODO(https://crbug.com/webrtc/11222): When the concept of "resources" that
+ // can be overused or underused has materialized, trigger overuse/underuse by
+ // injecting a fake Resource instead and remove these methods.
+ void OnResourceUnderuseForTesting(
+ AdaptationObserverInterface::AdaptReason reason);
+ bool OnResourceOveruseForTesting(
+ AdaptationObserverInterface::AdaptReason reason);
void OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions) override;
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index fcdac2e..472027b 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -166,9 +166,9 @@
rtc::Event event;
encoder_queue()->PostTask([this, &event, reason, down, expected_results] {
if (down)
- EXPECT_EQ(expected_results, TriggerAdaptDown(reason));
+ EXPECT_EQ(expected_results, OnResourceOveruseForTesting(reason));
else
- TriggerAdaptUp(reason);
+ OnResourceUnderuseForTesting(reason);
event.Set();
});
ASSERT_TRUE(event.Wait(5000));
