[Overuse] MaybeUpdateTargetFrameRate() & ResetVideoSourceRestrictions()
This CL does two things for the sake of getting us closer to adaptation
modules being injectable and usable without knowing implementation
details.
Firstly, RefreshTargetFramerate() is removed. The target frame rate is
dependent on two things: 1) the codec max frame rate, and 2) the video
source restrictions. If either of these two changes, the target frame
rate is updated - there is no need to trigger this externally; the
module already knows if either of these factors change.
The private method MaybeUpdateTargetFrameRate() is added to ensure
overuse_detector->OnTargetFramerateUpdated() happens when necessary.
In doing this, the frame rates are updated to use
absl::optional<double>. This documents its optionality and avoids
magical values (previously -1 was not a bug but meaning "missing"). It
also matches VideoSourceRestrictions::max_frame_rate()'s type.
Secondly, ResetAdaptationCounters() is renamed
ResetVideoSourceRestrictions(). This more accurately describes what it
is doing; it is resetting the restrictions (the adaptation counters
getting reset is merely an implementation specific side-effect of
this). This method is added to the generic interface.
The usefulness of being able to ResetVideoSourceRestrictions() is
questioned in a TODO - current usage of this is when "quality rampup"
finishes. Nevertheless, any module could implement this functionality
so it belongs to the interface for now.
Bug: webrtc:11222
Change-Id: I079785df55fc9894e85087ec98be3e4ebd0713c5
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/166522
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30320}
diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h
index e73adce..d71ffe8 100644
--- a/call/adaptation/resource_adaptation_module_interface.h
+++ b/call/adaptation/resource_adaptation_module_interface.h
@@ -61,6 +61,10 @@
virtual void SetHasInputVideo(bool has_input_video) = 0;
virtual void SetDegradationPreference(
DegradationPreference degradation_preference) = 0;
+ // Removes all restrictions; the module will need to adapt all over again.
+ // TODO(hbos): It's not clear why anybody should be able to tell the module to
+ // reset like this; can we get rid of this method?
+ virtual void ResetVideoSourceRestrictions() = 0;
};
} // namespace webrtc
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
index 20c1709..ea082a5 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.cc
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -355,7 +355,9 @@
last_frame_pixel_count_(absl::nullopt),
source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
overuse_detector_(std::move(overuse_detector)),
- codec_max_framerate_(-1),
+ overuse_detector_is_started_(false),
+ codec_max_frame_rate_(absl::nullopt),
+ target_frame_rate_(absl::nullopt),
encoder_start_bitrate_bps_(0),
is_quality_scaler_enabled_(false),
encoder_config_(),
@@ -378,6 +380,7 @@
void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation(
ResourceAdaptationModuleListener* adaptation_listener) {
RTC_DCHECK(encoder_);
+ RTC_DCHECK(!overuse_detector_is_started_);
// TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside
// the interval between StartCheckForOveruse() and StopCheckForOveruse(),
// support configuring which |adaptation_listener_| to use on the fly. It is
@@ -388,10 +391,16 @@
overuse_detector_->StartCheckForOveruse(
TaskQueueBase::Current(), video_stream_encoder_->GetCpuOveruseOptions(),
this);
+ overuse_detector_is_started_ = true;
+ overuse_detector_->OnTargetFramerateUpdated(
+ target_frame_rate_.has_value()
+ ? static_cast<int>(target_frame_rate_.value())
+ : std::numeric_limits<int>::max());
}
void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() {
overuse_detector_->StopCheckForOveruse();
+ overuse_detector_is_started_ = false;
}
void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo(
@@ -418,6 +427,14 @@
MaybeUpdateVideoSourceRestrictions();
}
+void OveruseFrameDetectorResourceAdaptationModule::
+ ResetVideoSourceRestrictions() {
+ last_adaptation_request_.reset();
+ source_restrictor_->ClearRestrictions();
+ adapt_counters_.clear();
+ MaybeUpdateVideoSourceRestrictions();
+}
+
void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured(
const VideoFrame& frame,
int64_t time_when_first_seen_us) {
@@ -443,9 +460,12 @@
encoder_config_ = std::move(encoder_config);
}
-void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFramerate(
- int codec_max_framerate) {
- codec_max_framerate_ = codec_max_framerate;
+void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFrameRate(
+ absl::optional<double> codec_max_frame_rate) {
+ RTC_DCHECK(!codec_max_frame_rate.has_value() ||
+ codec_max_frame_rate.value() > 0.0);
+ codec_max_frame_rate_ = codec_max_frame_rate;
+ MaybeUpdateTargetFrameRate();
}
void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps(
@@ -458,31 +478,6 @@
is_quality_scaler_enabled_ = is_quality_scaler_enabled;
}
-void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() {
- absl::optional<double> restricted_frame_rate =
- ApplyDegradationPreference(source_restrictor_->source_restrictions(),
- degradation_preference_)
- .max_frame_rate();
- // Get the current target framerate, ie the maximum framerate as specified by
- // the current codec configuration, or any limit imposed by cpu adaption in
- // maintain-resolution or balanced mode. This is used to make sure overuse
- // detection doesn't needlessly trigger in low and/or variable framerate
- // scenarios.
- int target_framerate =
- std::min(codec_max_framerate_,
- restricted_frame_rate.has_value()
- ? static_cast<int>(restricted_frame_rate.value())
- : std::numeric_limits<int>::max());
- overuse_detector_->OnTargetFramerateUpdated(target_framerate);
-}
-
-void OveruseFrameDetectorResourceAdaptationModule::ResetAdaptationCounters() {
- last_adaptation_request_.reset();
- source_restrictor_->ClearRestrictions();
- adapt_counters_.clear();
- MaybeUpdateVideoSourceRestrictions();
-}
-
void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
if (!has_input_video_)
return;
@@ -575,11 +570,8 @@
const int requested_framerate =
source_restrictor_->RequestHigherFramerateThan(fps);
if (requested_framerate == -1) {
- overuse_detector_->OnTargetFramerateUpdated(codec_max_framerate_);
return;
}
- overuse_detector_->OnTargetFramerateUpdated(
- std::min(codec_max_framerate_, requested_framerate));
GetAdaptCounter().DecrementFramerate(reason);
break;
}
@@ -683,9 +675,6 @@
adaptation_request.framerate_fps_);
if (requested_framerate == -1)
return true;
- RTC_DCHECK_NE(codec_max_framerate_, -1);
- overuse_detector_->OnTargetFramerateUpdated(
- std::min(codec_max_framerate_, requested_framerate));
GetAdaptCounter().IncrementFramerate(reason);
break;
}
@@ -713,6 +702,33 @@
video_source_restrictions_ = std::move(new_restrictions);
adaptation_listener_->OnVideoSourceRestrictionsUpdated(
video_source_restrictions_);
+ MaybeUpdateTargetFrameRate();
+ }
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::
+ MaybeUpdateTargetFrameRate() {
+ // The current target framerate is the maximum frame rate as specified by
+ // the current codec configuration or any limit imposed by the adaptation
+ // module. This is used to make sure overuse detection doesn't needlessly
+ // trigger in low and/or variable framerate scenarios.
+ absl::optional<double> target_frame_rate =
+ ApplyDegradationPreference(source_restrictor_->source_restrictions(),
+ degradation_preference_)
+ .max_frame_rate();
+ if (!target_frame_rate.has_value() ||
+ (codec_max_frame_rate_.has_value() &&
+ codec_max_frame_rate_.value() < target_frame_rate.value())) {
+ target_frame_rate = codec_max_frame_rate_;
+ }
+ if (target_frame_rate != target_frame_rate_) {
+ target_frame_rate_ = target_frame_rate;
+ if (overuse_detector_is_started_) {
+ overuse_detector_->OnTargetFramerateUpdated(
+ target_frame_rate_.has_value()
+ ? static_cast<int>(target_frame_rate_.value())
+ : std::numeric_limits<int>::max());
+ }
}
}
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
index 82f0c27..d3b88e5 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.h
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -77,6 +77,7 @@
void SetHasInputVideo(bool has_input_video) override;
void SetDegradationPreference(
DegradationPreference degradation_preference) override;
+ void ResetVideoSourceRestrictions() override;
// Input to the OveruseFrameDetector, which are required for this module to
// function. These map to OveruseFrameDetector methods.
@@ -95,7 +96,7 @@
// need this many public methods?
void SetLastFramePixelCount(absl::optional<int> last_frame_pixel_count);
void SetEncoderConfig(VideoEncoderConfig encoder_config);
- void SetCodecMaxFramerate(int codec_max_framerate);
+ void SetCodecMaxFrameRate(absl::optional<double> codec_max_frame_rate);
void SetEncoderStartBitrateBps(uint32_t encoder_start_bitrate_bps);
// Inform the detector whether or not the quality scaler is enabled. This
// helps GetActiveCounts() return absl::nullopt when appropriate.
@@ -104,11 +105,6 @@
// method is called incorrectly.
void SetIsQualityScalerEnabled(bool is_quality_scaler_enabled);
- // TODO(hbos): Can we get rid of this? Seems we should know whether the frame
- // rate has updated.
- void RefreshTargetFramerate();
- void ResetAdaptationCounters();
-
class AdaptCounter final {
public:
AdaptCounter();
@@ -189,6 +185,10 @@
// |adaptation_listener_| if restrictions are changed, allowing the listener
// to reconfigure the source accordingly.
void MaybeUpdateVideoSourceRestrictions();
+ // Calculates an up-to-date value of |target_frame_rate_| and informs the
+ // |overuse_detector_| of the new value if it changed and the detector is
+ // started.
+ void MaybeUpdateTargetFrameRate();
void UpdateAdaptationStats(AdaptReason reason);
DegradationPreference EffectiveDegradataionPreference();
@@ -216,7 +216,9 @@
// Keeps track of source restrictions that this adaptation module outputs.
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
const std::unique_ptr<OveruseFrameDetector> overuse_detector_;
- int codec_max_framerate_;
+ bool overuse_detector_is_started_;
+ absl::optional<double> codec_max_frame_rate_;
+ absl::optional<double> target_frame_rate_;
uint32_t encoder_start_bitrate_bps_;
bool is_quality_scaler_enabled_;
VideoEncoderConfig encoder_config_;
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 6d603f3..52dc53f 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -78,11 +78,6 @@
degradation_preference == DegradationPreference::BALANCED;
}
-bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
- return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION ||
- degradation_preference == DegradationPreference::BALANCED;
-}
-
bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
const VideoCodec& new_send_codec,
bool was_encode_called_since_last_initialization) {
@@ -393,13 +388,6 @@
degradation_preference);
if (encoder_)
ConfigureQualityScaler(encoder_->GetEncoderInfo());
-
- if (!IsFramerateScalingEnabled(degradation_preference) &&
- max_framerate_ != -1) {
- // If frame rate scaling is no longer allowed, remove any potential
- // allowance for longer frame intervals.
- resource_adaptation_module_->RefreshTargetFramerate();
- }
});
}
@@ -599,7 +587,7 @@
// Make sure the start bit rate is sane...
RTC_DCHECK_LE(codec.startBitrate, 1000000);
max_framerate_ = codec.maxFramerate;
- resource_adaptation_module_->SetCodecMaxFramerate(max_framerate_);
+ resource_adaptation_module_->SetCodecMaxFrameRate(max_framerate_);
// Inform source about max configured framerate.
int max_framerate = 0;
@@ -750,8 +738,6 @@
std::move(streams), encoder_config_.content_type,
encoder_config_.min_transmit_bitrate_bps);
- resource_adaptation_module_->RefreshTargetFramerate();
-
ConfigureQualityScaler(info);
}
@@ -1137,7 +1123,7 @@
resource_adaptation_module_->GetConstAdaptCounter().TotalCount(
AdaptationObserverInterface::AdaptReason::kCpu) == 0) {
RTC_LOG(LS_INFO) << "Reset quality limitations.";
- resource_adaptation_module_->ResetAdaptationCounters();
+ resource_adaptation_module_->ResetVideoSourceRestrictions();
quality_rampup_done_ = true;
}
