Move DegradationPreference logic to the encoder queue.
This moves SetHasInputVideoAndDegradationPreference() to the encoder
queue. OveruseFrameDetectorResourceAdaptationModule is now entirely
single-threaded, including its inner class VideoSourceRestrictor.
VideoStreamEncoder now protects the module with RTC_GUARDED_BY. This
ensures it is safely used, even without a SequenceChecker inside of the
module. The module's |encoder_queue_| is removed.
The one task queue reference that is needed - passing down the current
task queue to StartCheckForOveruse() - is replaced by a TaskQueueBase*
(instead of rtc::TaskQueue*), enabling obtaining the current queue with
TaskQueueBase::Current(). (There is no rtc::TaskQueue::Current().)
Furthermore, the only uses of VideoSourceSinkController that isn't on
the encoder queue are documented, with a TODO saying if these are moved
the VideoSourceSinkController could also be made single-threaded.
However since this requires introducing a delay to
VideoStreamEncoder::SetSource() and VideoStreamEncoder::Stop(),
arguably a more risky change, if this is to be attempted that should be
in a separate CL.
Bug: webrtc:11222
Change-Id: I448ca5125708d5f66b95b0b180d6d24cc356dfa9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/165783
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30263}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index c77973e..a048a2b 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -194,6 +194,7 @@
deps = [
"../api:rtp_parameters",
+ "../api/task_queue:task_queue",
"../api/units:data_rate",
"../api/video:encoded_image",
"../api/video:video_bitrate_allocation",
diff --git a/video/overuse_frame_detector.cc b/video/overuse_frame_detector.cc
index ade9303..1046c1e 100644
--- a/video/overuse_frame_detector.cc
+++ b/video/overuse_frame_detector.cc
@@ -540,7 +540,7 @@
OveruseFrameDetector::~OveruseFrameDetector() {}
void OveruseFrameDetector::StartCheckForOveruse(
- rtc::TaskQueue* task_queue,
+ TaskQueueBase* task_queue_base,
const CpuOveruseOptions& options,
AdaptationObserverInterface* overuse_observer) {
RTC_DCHECK_RUN_ON(&task_checker_);
@@ -549,7 +549,7 @@
SetOptions(options);
check_overuse_task_ = RepeatingTaskHandle::DelayedStart(
- task_queue->Get(), TimeDelta::ms(kTimeToFirstCheckForOveruseMs),
+ task_queue_base, TimeDelta::ms(kTimeToFirstCheckForOveruseMs),
[this, overuse_observer] {
CheckForOveruse(overuse_observer);
return TimeDelta::ms(kCheckForOveruseIntervalMs);
diff --git a/video/overuse_frame_detector.h b/video/overuse_frame_detector.h
index 20e0971..4f64734 100644
--- a/video/overuse_frame_detector.h
+++ b/video/overuse_frame_detector.h
@@ -15,13 +15,13 @@
#include <memory>
#include "absl/types/optional.h"
+#include "api/task_queue/task_queue_base.h"
#include "api/video/video_stream_encoder_observer.h"
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/experiments/field_trial_parser.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/synchronization/sequence_checker.h"
-#include "rtc_base/task_queue.h"
#include "rtc_base/task_utils/repeating_task.h"
#include "rtc_base/thread_annotations.h"
@@ -58,7 +58,7 @@
virtual ~OveruseFrameDetector();
// Start to periodically check for overuse.
- void StartCheckForOveruse(rtc::TaskQueue* task_queue,
+ void StartCheckForOveruse(TaskQueueBase* task_queue_base,
const CpuOveruseOptions& options,
AdaptationObserverInterface* overuse_observer);
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
index 1e449b9..2bd937b 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.cc
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -17,6 +17,7 @@
#include <utility>
#include "absl/algorithm/container.h"
+#include "api/task_queue/task_queue_base.h"
#include "api/video/video_source_interface.h"
#include "call/adaptation/video_source_restrictions.h"
#include "rtc_base/logging.h"
@@ -73,10 +74,6 @@
// AdaptDown() - only how to modify the source/sink restrictions when this
// happens. Note that it is also not responsible for reconfigruring the
// source/sink, it is only a keeper of desired restrictions.
-//
-// Thread safety is ensured between SetHasInputVideoAndDegradationPreference()
-// calls on the worker thread and adaptation logic on the encoder task queue
-// using a lock.
class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
public:
VideoSourceRestrictor()
@@ -84,25 +81,19 @@
degradation_preference_(DegradationPreference::DISABLED) {}
VideoSourceRestrictions source_restrictions() {
- rtc::CritScope lock(&crit_);
return source_restrictions_;
}
- // Inform the restrictor of new source status and degradation preference.
- // TODO(hbos): Can this be moved to the encoder queue? If so, the |crit_| lock
- // can be removed and we only need a sequence checker.
void SetHasInputVideoAndDegradationPreference(
bool has_input_video,
DegradationPreference degradation_preference) {
- // Called on libjingle's worker thread.
- RTC_DCHECK_RUN_ON(&main_checker_);
- rtc::CritScope lock(&crit_);
has_input_video_ = has_input_video;
degradation_preference_ = degradation_preference;
}
+ // Updates the source_restrictions(). The source/sink has to be informed of
+ // this separately.
void ClearRestrictions() {
- rtc::CritScope lock(&crit_);
source_restrictions_ = VideoSourceRestrictions();
}
@@ -111,14 +102,9 @@
bool RequestResolutionLowerThan(int pixel_count,
int min_pixels_per_frame,
bool* min_pixels_reached) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!has_input_video_ ||
- !IsResolutionScalingEnabled(degradation_preference_)) {
- // This can happen since |degradation_preference_| is set on libjingle's
- // worker thread but the adaptation is done on the encoder task queue.
+ RTC_DCHECK(IsResolutionScalingEnabled(degradation_preference_));
+ if (!has_input_video_)
return false;
- }
// The input video frame size will have a resolution less than or equal to
// |max_pixel_count| depending on how the source can scale the frame size.
const int pixels_wanted = (pixel_count * 3) / 5;
@@ -145,7 +131,6 @@
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
int RequestFramerateLowerThan(int fps) {
- // Called on the encoder task queue.
// The input video frame rate will be scaled down to 2/3, rounding down.
int framerate_wanted = (fps * 2) / 3;
return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
@@ -164,14 +149,9 @@
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
bool RequestHigherResolutionThan(int pixel_count) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!has_input_video_ ||
- !IsResolutionScalingEnabled(degradation_preference_)) {
- // This can happen since |degradation_preference_| is set on libjingle's
- // worker thread but the adaptation is done on the encoder task queue.
+ RTC_DCHECK(IsResolutionScalingEnabled(degradation_preference_));
+ if (!has_input_video_)
return false;
- }
int max_pixels_wanted = pixel_count;
if (max_pixels_wanted != std::numeric_limits<int>::max())
max_pixels_wanted = pixel_count * 4;
@@ -203,7 +183,6 @@
// adaptation requests are removed completely. In that case, consider
// |max_framerate_| to be the current limit (assuming the capturer complies).
int RequestHigherFramerateThan(int fps) {
- // Called on the encoder task queue.
// The input frame rate will be scaled up to the last step, with rounding.
int framerate_wanted = fps;
if (fps != std::numeric_limits<int>::max())
@@ -215,10 +194,8 @@
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
bool RestrictFramerate(int fps) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!has_input_video_ ||
- !IsFramerateScalingEnabled(degradation_preference_))
+ RTC_DCHECK(IsFramerateScalingEnabled(degradation_preference_));
+ if (!has_input_video_)
return false;
const int fps_wanted = std::max(kMinFramerateFps, fps);
@@ -238,10 +215,8 @@
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
bool IncreaseFramerate(int fps) {
- // Called on the encoder task queue.
- rtc::CritScope lock(&crit_);
- if (!has_input_video_ ||
- !IsFramerateScalingEnabled(degradation_preference_))
+ RTC_DCHECK(IsFramerateScalingEnabled(degradation_preference_));
+ if (!has_input_video_)
return false;
const int fps_wanted = std::max(kMinFramerateFps, fps);
@@ -259,11 +234,9 @@
}
private:
- rtc::CriticalSection crit_;
- SequenceChecker main_checker_;
- VideoSourceRestrictions source_restrictions_ RTC_GUARDED_BY(&crit_);
- bool has_input_video_ RTC_GUARDED_BY(&crit_);
- DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_);
+ VideoSourceRestrictions source_restrictions_;
+ bool has_input_video_;
+ DegradationPreference degradation_preference_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
};
@@ -397,8 +370,7 @@
std::unique_ptr<OveruseFrameDetector> overuse_detector,
VideoStreamEncoderObserver* encoder_stats_observer,
ResourceAdaptationModuleListener* adaptation_listener)
- : encoder_queue_(nullptr),
- adaptation_listener_(adaptation_listener),
+ : adaptation_listener_(adaptation_listener),
video_stream_encoder_(video_stream_encoder),
degradation_preference_(DegradationPreference::DISABLED),
adapt_counters_(),
@@ -422,24 +394,13 @@
OveruseFrameDetectorResourceAdaptationModule::
~OveruseFrameDetectorResourceAdaptationModule() {}
-void OveruseFrameDetectorResourceAdaptationModule::Initialize(
- rtc::TaskQueue* encoder_queue) {
- RTC_DCHECK(!encoder_queue_);
- encoder_queue_ = encoder_queue;
- RTC_DCHECK(encoder_queue_);
-}
-
void OveruseFrameDetectorResourceAdaptationModule::SetEncoder(
VideoEncoder* encoder) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_ = encoder;
}
void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse(
ResourceAdaptationModuleListener* adaptation_listener) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(encoder_);
// TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside
// the interval between StartCheckForOveruse() and StopCheckForOveruse(),
@@ -449,20 +410,17 @@
// AdaptUp() and AdaptDown() even when the OveruseDetector is inactive.
RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_);
overuse_detector_->StartCheckForOveruse(
- encoder_queue_, video_stream_encoder_->GetCpuOveruseOptions(), this);
+ TaskQueueBase::Current(), video_stream_encoder_->GetCpuOveruseOptions(),
+ this);
}
void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
overuse_detector_->StopCheckForOveruse();
}
void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured(
const VideoFrame& frame,
int64_t time_when_first_seen_us) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
overuse_detector_->FrameCaptured(frame, time_when_first_seen_us);
}
@@ -471,44 +429,32 @@
int64_t time_sent_in_us,
int64_t capture_time_us,
absl::optional<int> encode_duration_us) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us,
encode_duration_us);
}
void OveruseFrameDetectorResourceAdaptationModule::SetLastFramePixelCount(
absl::optional<int> last_frame_pixel_count) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
last_frame_pixel_count_ = last_frame_pixel_count;
}
void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig(
VideoEncoderConfig encoder_config) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_config_ = std::move(encoder_config);
}
void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFramerate(
int codec_max_framerate) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
codec_max_framerate_ = codec_max_framerate;
}
void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps(
uint32_t encoder_start_bitrate_bps) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_start_bitrate_bps_ = encoder_start_bitrate_bps;
}
void OveruseFrameDetectorResourceAdaptationModule::SetIsQualityScalerEnabled(
bool is_quality_scaler_enabled) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
is_quality_scaler_enabled_ = is_quality_scaler_enabled;
}
@@ -516,43 +462,25 @@
SetHasInputVideoAndDegradationPreference(
bool has_input_video,
DegradationPreference degradation_preference) {
- // TODO(https://crbug.com/webrtc/11222): Move this call to the encoder queue,
- // making VideoSourceRestrictor single-threaded and removing the only call to
- // MaybeUpdateVideoSourceRestrictions() that isn't on the |encoder_queue_|.
- source_restrictor_->SetHasInputVideoAndDegradationPreference(
- has_input_video, degradation_preference);
- MaybeUpdateVideoSourceRestrictions(degradation_preference);
- encoder_queue_->PostTask([this, degradation_preference] {
- RTC_DCHECK_RUN_ON(encoder_queue_);
- if (degradation_preference_ != degradation_preference) {
- // Reset adaptation state, so that we're not tricked into thinking there's
- // an already pending request of the same type.
- last_adaptation_request_.reset();
- if (degradation_preference == DegradationPreference::BALANCED ||
- degradation_preference_ == DegradationPreference::BALANCED) {
- // TODO(asapersson): Consider removing |adapt_counters_| map and use one
- // AdaptCounter for all modes.
- source_restrictor_->ClearRestrictions();
- adapt_counters_.clear();
- }
+ if (degradation_preference_ != degradation_preference) {
+ // Reset adaptation state, so that we're not tricked into thinking there's
+ // an already pending request of the same type.
+ last_adaptation_request_.reset();
+ if (degradation_preference == DegradationPreference::BALANCED ||
+ degradation_preference_ == DegradationPreference::BALANCED) {
+ // TODO(asapersson): Consider removing |adapt_counters_| map and use one
+ // AdaptCounter for all modes.
+ source_restrictor_->ClearRestrictions();
+ adapt_counters_.clear();
}
- degradation_preference_ = degradation_preference;
- // This is the second time we're invoking
- // MaybeUpdateVideoSourceRestrictions() in this method. This is because
- // current tests expect the changes to the source restrictions to be
- // immediate (outside of the encoder queue) while it is possible that they
- // change again after ClearRestrictions() on the encoder queue.
- // TODO(https://crbug.com/webrtc/11222): Change the expectations to allow
- // source restrictions only to change on the encoder queue. This unblocks
- // making OveruseFrameDetectorResourceAdaptationModule and
- // VideoSourceRestrictor single-threaded.
- MaybeUpdateVideoSourceRestrictions(degradation_preference_);
- });
+ }
+ degradation_preference_ = degradation_preference;
+ source_restrictor_->SetHasInputVideoAndDegradationPreference(
+ has_input_video, degradation_preference_);
+ MaybeUpdateVideoSourceRestrictions();
}
void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
absl::optional<double> restricted_frame_rate =
ApplyDegradationPreference(source_restrictor_->source_restrictions(),
degradation_preference_)
@@ -571,17 +499,13 @@
}
void OveruseFrameDetectorResourceAdaptationModule::ResetAdaptationCounters() {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
last_adaptation_request_.reset();
source_restrictor_->ClearRestrictions();
adapt_counters_.clear();
- MaybeUpdateVideoSourceRestrictions(degradation_preference_);
+ MaybeUpdateVideoSourceRestrictions();
}
void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
const AdaptCounter& adapt_counter = GetConstAdaptCounter();
int num_downgrades = adapt_counter.TotalCount(reason);
if (num_downgrades == 0)
@@ -685,7 +609,7 @@
// Tell the adaptation listener to reconfigure the source for us according to
// the latest adaptation.
- MaybeUpdateVideoSourceRestrictions(degradation_preference_);
+ MaybeUpdateVideoSourceRestrictions();
last_adaptation_request_.emplace(adaptation_request);
@@ -696,8 +620,6 @@
bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
AdaptReason reason) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
AdaptationRequest adaptation_request = {
*last_frame_pixel_count_, encoder_stats_observer_->GetInputFrameRate(),
AdaptationRequest::Mode::kAdaptDown};
@@ -791,7 +713,7 @@
// Tell the adaptation listener to reconfigure the source for us according to
// the latest adaptation.
- MaybeUpdateVideoSourceRestrictions(degradation_preference_);
+ MaybeUpdateVideoSourceRestrictions();
last_adaptation_request_.emplace(adaptation_request);
@@ -802,21 +724,13 @@
}
void OveruseFrameDetectorResourceAdaptationModule::
- MaybeUpdateVideoSourceRestrictions(
- DegradationPreference degradation_preference) {
- absl::optional<VideoSourceRestrictions> updated_restrictions;
- {
- rtc::CritScope lock(&video_source_restrictions_crit_);
- VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
- source_restrictor_->source_restrictions(), degradation_preference);
- if (video_source_restrictions_ != new_restrictions) {
- video_source_restrictions_ = std::move(new_restrictions);
- updated_restrictions = video_source_restrictions_;
- }
- }
- if (updated_restrictions.has_value()) {
+ MaybeUpdateVideoSourceRestrictions() {
+ VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
+ source_restrictor_->source_restrictions(), degradation_preference_);
+ if (video_source_restrictions_ != new_restrictions) {
+ video_source_restrictions_ = std::move(new_restrictions);
adaptation_listener_->OnVideoSourceRestrictionsUpdated(
- updated_restrictions.value());
+ video_source_restrictions_);
}
}
@@ -840,8 +754,6 @@
VideoStreamEncoderObserver::AdaptationSteps
OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
AdaptReason reason) {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
VideoStreamEncoderObserver::AdaptationSteps counts =
GetConstAdaptCounter().Counts(reason);
switch (reason) {
@@ -885,15 +797,11 @@
const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter&
OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
return adapt_counters_[degradation_preference_];
}
absl::optional<VideoEncoder::QpThresholds>
OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(last_frame_pixel_count_.has_value());
return balanced_settings_.GetQpThresholds(encoder_config_.codec_type,
last_frame_pixel_count_.value());
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
index dcd78e8..28902cf 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.h
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -36,6 +36,9 @@
// resolution up or down based on encode usage percent. It keeps track of video
// source settings, adaptation counters and may get influenced by
// VideoStreamEncoder's quality scaler through AdaptUp() and AdaptDown() calls.
+//
+// This class is single-threaded. The caller is responsible for ensuring safe
+// usage.
// TODO(hbos): Reduce the coupling with VideoStreamEncoder.
// TODO(hbos): Add unittests specific to this class, it is currently only tested
// indirectly in video_stream_encoder_unittest.cc and other tests exercising
@@ -48,6 +51,8 @@
: public ResourceAdaptationModuleInterface,
public AdaptationObserverInterface {
public:
+ // The module can be constructed on any sequence, but must be initialized and
+ // used on a single sequence, e.g. the encoder queue.
OveruseFrameDetectorResourceAdaptationModule(
VideoStreamEncoder* video_stream_encoder,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
@@ -55,7 +60,6 @@
ResourceAdaptationModuleListener* adaptation_listener);
~OveruseFrameDetectorResourceAdaptationModule() override;
- void Initialize(rtc::TaskQueue* encoder_queue);
// Sets the encoder to reconfigure based on overuse.
// TODO(hbos): Don't reconfigure the encoder directly. Instead, define the
// output of a resource adaptation module as a struct and let the
@@ -63,8 +67,6 @@
void SetEncoder(VideoEncoder* encoder);
DegradationPreference degradation_preference() const {
- RTC_DCHECK(encoder_queue_);
- RTC_DCHECK_RUN_ON(encoder_queue_);
return degradation_preference_;
}
@@ -187,60 +189,39 @@
// Makes |video_source_restrictions_| up-to-date and informs the
// |adaptation_listener_| if restrictions are changed, allowing the listener
// to reconfigure the source accordingly.
- // TODO(https://crbug.com/webrtc/11222): When
- // SetHasInputVideoAndDegradationPreference() stops calling this method prior
- // to updating |degradation_preference_| on the encoder queue, remove its
- // argument in favor of using |degradation_preference_| directly.
- void MaybeUpdateVideoSourceRestrictions(
- DegradationPreference degradation_preference);
+ void MaybeUpdateVideoSourceRestrictions();
- void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(encoder_queue_);
- DegradationPreference EffectiveDegradataionPreference()
- RTC_RUN_ON(encoder_queue_);
- AdaptCounter& GetAdaptCounter() RTC_RUN_ON(encoder_queue_);
- bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const
- RTC_RUN_ON(encoder_queue_);
+ void UpdateAdaptationStats(AdaptReason reason);
+ DegradationPreference EffectiveDegradataionPreference();
+ AdaptCounter& GetAdaptCounter();
+ bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const;
- rtc::TaskQueue* encoder_queue_;
- // TODO(https://crbug.com/webrtc/11222): Update
- // SetHasInputVideoAndDegradationPreference() to do all work on the encoder
- // queue (including |source_restrictor_| and |adaptation_listener_| usage).
- // When this is the case, remove |VideoSourceRestrictor::crit_| and
- // |video_source_restrictions_crit_| and replace |encoder_queue_| with a
- // sequence checker.
- rtc::CriticalSection video_source_restrictions_crit_;
ResourceAdaptationModuleListener* const adaptation_listener_;
// The restrictions that |adaptation_listener_| is informed of.
- VideoSourceRestrictions video_source_restrictions_
- RTC_GUARDED_BY(&video_source_restrictions_crit_);
+ VideoSourceRestrictions video_source_restrictions_;
// Used to query CpuOveruseOptions at StartCheckForOveruse().
- VideoStreamEncoder* video_stream_encoder_ RTC_GUARDED_BY(encoder_queue_);
- DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_);
+ VideoStreamEncoder* video_stream_encoder_;
+ DegradationPreference degradation_preference_;
// Counters used for deciding if the video resolution or framerate is
// currently restricted, and if so, why, on a per degradation preference
// basis.
// TODO(sprang): Replace this with a state holding a relative overuse measure
// instead, that can be translated into suitable down-scale or fps limit.
- std::map<const DegradationPreference, AdaptCounter> adapt_counters_
- RTC_GUARDED_BY(encoder_queue_);
- const BalancedDegradationSettings balanced_settings_
- RTC_GUARDED_BY(encoder_queue_);
+ std::map<const DegradationPreference, AdaptCounter> adapt_counters_;
+ const BalancedDegradationSettings balanced_settings_;
// Stores a snapshot of the last adaptation request triggered by an AdaptUp
// or AdaptDown signal.
- absl::optional<AdaptationRequest> last_adaptation_request_
- RTC_GUARDED_BY(encoder_queue_);
- absl::optional<int> last_frame_pixel_count_ RTC_GUARDED_BY(encoder_queue_);
+ absl::optional<AdaptationRequest> last_adaptation_request_;
+ absl::optional<int> last_frame_pixel_count_;
// Keeps track of source restrictions that this adaptation module outputs.
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
- const std::unique_ptr<OveruseFrameDetector> overuse_detector_
- RTC_PT_GUARDED_BY(encoder_queue_);
- int codec_max_framerate_ RTC_GUARDED_BY(encoder_queue_);
- uint32_t encoder_start_bitrate_bps_ RTC_GUARDED_BY(encoder_queue_);
- bool is_quality_scaler_enabled_ RTC_GUARDED_BY(encoder_queue_);
- VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(encoder_queue_);
- VideoEncoder* encoder_ RTC_GUARDED_BY(encoder_queue_);
- VideoStreamEncoderObserver* const encoder_stats_observer_
- RTC_GUARDED_BY(encoder_queue_);
+ const std::unique_ptr<OveruseFrameDetector> overuse_detector_;
+ int codec_max_framerate_;
+ uint32_t encoder_start_bitrate_bps_;
+ bool is_quality_scaler_enabled_;
+ VideoEncoderConfig encoder_config_;
+ VideoEncoder* encoder_;
+ VideoStreamEncoderObserver* const encoder_stats_observer_;
};
} // namespace webrtc
diff --git a/video/overuse_frame_detector_unittest.cc b/video/overuse_frame_detector_unittest.cc
index 7d8217c..432cc27 100644
--- a/video/overuse_frame_detector_unittest.cc
+++ b/video/overuse_frame_detector_unittest.cc
@@ -433,7 +433,8 @@
queue.SendTask(
[&] {
- overuse_detector_->StartCheckForOveruse(&queue, options_, observer_);
+ overuse_detector_->StartCheckForOveruse(queue.Get(), options_,
+ observer_);
},
RTC_FROM_HERE);
@@ -914,7 +915,8 @@
queue.SendTask(
[&] {
- overuse_detector_->StartCheckForOveruse(&queue, options_, observer_);
+ overuse_detector_->StartCheckForOveruse(queue.Get(), options_,
+ observer_);
},
RTC_FROM_HERE);
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 3ddbf58..eecd7de 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -332,7 +332,6 @@
TaskQueueFactory::Priority::NORMAL)) {
RTC_DCHECK(encoder_stats_observer);
RTC_DCHECK_GE(number_of_cores, 1);
- resource_adaptation_module_->Initialize(encoder_queue());
for (auto& state : encoder_buffer_state_)
state.fill(std::numeric_limits<int64_t>::max());
@@ -388,10 +387,10 @@
const DegradationPreference& degradation_preference) {
RTC_DCHECK_RUN_ON(&thread_checker_);
video_source_sink_controller_->SetSource(source);
- resource_adaptation_module_->SetHasInputVideoAndDegradationPreference(
- source, degradation_preference);
- encoder_queue_.PostTask([this, degradation_preference] {
+ encoder_queue_.PostTask([this, source, degradation_preference] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
+ resource_adaptation_module_->SetHasInputVideoAndDegradationPreference(
+ source, degradation_preference);
if (encoder_)
ConfigureQualityScaler(encoder_->GetEncoderInfo());
@@ -1731,19 +1730,19 @@
bool VideoStreamEncoder::TriggerAdaptDown(
AdaptationObserverInterface::AdaptReason reason) {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
return resource_adaptation_module_->AdaptDown(reason);
}
void VideoStreamEncoder::TriggerAdaptUp(
AdaptationObserverInterface::AdaptReason reason) {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
resource_adaptation_module_->AdaptUp(reason);
}
void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions) {
- // TODO(https://crbug.com/webrtc/11222): DCHECK that we are using the
- // |encoder_queue_| when OnVideoSourceRestrictionsUpdated() is no longer
- // invoked off this thread due to VideoStreamEncoder::SetSource() stuff.
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
video_source_sink_controller_->SetRestrictions(std::move(restrictions));
video_source_sink_controller_->PushSourceSinkSettings();
}
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 6db3d53..bd76ee5 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -408,9 +408,20 @@
// track of whether a request has been made or not.
bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_);
+ // The controller updates the sink wants based on restrictions that come from
+ // the resource adaptation module or adaptation due to bandwidth adaptation.
+ //
+ // This is used on the encoder queue, with a few exceptions:
+ // - VideoStreamEncoder::SetSource() invokes SetSource().
+ // - VideoStreamEncoder::SetSink() invokes SetRotationApplied() and
+ // PushSourceSinkSettings().
+ // - VideoStreamEncoder::Stop() invokes SetSource().
+ // TODO(hbos): If these can be moved to the encoder queue,
+ // VideoSourceSinkController can be made single-threaded, and its lock can be
+ // replaced with a sequence checker.
std::unique_ptr<VideoSourceSinkController> video_source_sink_controller_;
std::unique_ptr<OveruseFrameDetectorResourceAdaptationModule>
- resource_adaptation_module_;
+ resource_adaptation_module_ RTC_GUARDED_BY(&encoder_queue_);
// All public methods are proxied to |encoder_queue_|. It must must be
// destroyed first to make sure no tasks are run that use other members.
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 4b4cd2d..e5439f3 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -1805,7 +1805,12 @@
test::FrameForwarder new_video_source;
video_stream_encoder_->SetSource(
&new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
-
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
+ sink_.WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
// Initially no degradation registered.
VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
@@ -1831,6 +1836,12 @@
// Turn off degradation completely.
video_stream_encoder_->SetSource(&new_video_source,
webrtc::DegradationPreference::DISABLED);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
+ sink_.WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
video_stream_encoder_->TriggerCpuOveruse();
@@ -1845,6 +1856,12 @@
// Calling SetSource with resolution scaling enabled apply the old SinkWants.
video_stream_encoder_->SetSource(
&new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
+ sink_.WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
kFrameWidth * kFrameHeight);
EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
@@ -1853,6 +1870,12 @@
// Calling SetSource with framerate scaling enabled apply the old SinkWants.
video_stream_encoder_->SetSource(
&new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
+ sink_.WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
EXPECT_EQ(std::numeric_limits<int>::max(),
new_video_source.sink_wants().max_pixel_count);
@@ -2561,12 +2584,16 @@
test::FrameForwarder new_video_source;
video_stream_encoder_->SetSource(
&new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(3);
VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
// Trigger adapt down, expect reduced framerate.
video_stream_encoder_->TriggerQualityLow();
- new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
- sink_.WaitForEncodedFrame(3);
+ new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(4);
VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
// Trigger adapt up, expect no restriction.