[Adaptation] Make Resources reference counted and add more DCHECKs.
In a future CL, adaptation processing and stream encoder resource
management will happen on different task queues. When this is the case,
asynchronous tasks will be posted in both directions and some resources
will have internal states used on multiple threads.
This CL makes the Resource class reference counted in order to support
posting tasks to a different threads without risk of use-after-free
when a posted task is executed with a delay. This is preferred over
WeakPtr strategies because WeakPtrs are single-threaded and preferred
over raw pointer usage because the reference counted approach enables
more compile-time and run-time assurance. This is also "future proof";
when resources can be injected through public APIs, ownership needs to
be shared between libwebrtc and the application (e.g. Chrome).
To reduce the risk of making mistakes in the future CL, sequence
checkers and task queue DCHECKs are added as well as other DCHECKs to
make sure things have been cleaned up before destruction, e.g:
- Processor gets a sequence checker. It is entirely single-threaded.
- Processor must not have any attached listeners or resources on
destruction.
- Resources must not have any listeners on destruction.
- The Manager, EncodeUsageResource and QualityScalerResource DCHECKs
they are running on the encoder queue.
- TODOs are added illustrating where we want to add PostTasks in the
future CL.
Lastly, upon VideoStreamEncoder::Stop() we delete the
ResourceAdaptationProcessor. Because the Processor is already used in
posted tasks, some if statements are added to ensure the Processor is
not used after destruction.
Bug: webrtc:11542, webrtc:11520
Change-Id: Ibaa8a61d86d87a71f477d1075a117c28d9d2d285
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/174760
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31217}
diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn
index deac315..c54665c 100644
--- a/call/adaptation/BUILD.gn
+++ b/call/adaptation/BUILD.gn
@@ -29,6 +29,7 @@
]
deps = [
"../../api:rtp_parameters",
+ "../../api:scoped_refptr",
"../../api/video:video_adaptation",
"../../api/video:video_frame",
"../../api/video:video_stream_encoder",
@@ -36,7 +37,9 @@
"../../modules/video_coding:video_coding_utility",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
+ "../../rtc_base:rtc_task_queue",
"../../rtc_base/experiments:balanced_degradation_settings",
+ "../../rtc_base/synchronization:sequence_checker",
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/types:optional",
]
@@ -56,6 +59,7 @@
deps = [
":resource_adaptation",
":resource_adaptation_test_utilities",
+ "../../api:scoped_refptr",
"../../api/video:video_adaptation",
"../../api/video_codecs:video_codecs_api",
"../../rtc_base:checks",
diff --git a/call/adaptation/resource.cc b/call/adaptation/resource.cc
index 1f90934..52343ee 100644
--- a/call/adaptation/resource.cc
+++ b/call/adaptation/resource.cc
@@ -19,7 +19,10 @@
Resource::Resource() : usage_state_(absl::nullopt), listener_(nullptr) {}
-Resource::~Resource() {}
+Resource::~Resource() {
+ RTC_DCHECK(!listener_)
+ << "There is a listener depending on a Resource being destroyed.";
+}
void Resource::SetResourceListener(ResourceListener* listener) {
// If you want to change listener you need to unregister the old listener by
@@ -40,7 +43,7 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const {
+ rtc::scoped_refptr<Resource> reason_resource) const {
return true;
}
@@ -48,13 +51,13 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) {}
+ rtc::scoped_refptr<Resource> reason_resource) {}
void Resource::OnResourceUsageStateMeasured(ResourceUsageState usage_state) {
usage_state_ = usage_state;
if (!listener_)
return;
- listener_->OnResourceUsageStateMeasured(*this);
+ listener_->OnResourceUsageStateMeasured(this);
}
} // namespace webrtc
diff --git a/call/adaptation/resource.h b/call/adaptation/resource.h
index 1f58dc1..6c7af48 100644
--- a/call/adaptation/resource.h
+++ b/call/adaptation/resource.h
@@ -15,8 +15,10 @@
#include <vector>
#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_input_state.h"
+#include "rtc_base/ref_count.h"
namespace webrtc {
@@ -34,15 +36,16 @@
virtual ~ResourceListener();
// Informs the listener of a new measurement of resource usage. This means
- // that |resource.usage_state()| is now up-to-date.
- virtual void OnResourceUsageStateMeasured(const Resource& resource) = 0;
+ // that |resource->usage_state()| is now up-to-date.
+ virtual void OnResourceUsageStateMeasured(
+ rtc::scoped_refptr<Resource> resource) = 0;
};
-class Resource {
+class Resource : public rtc::RefCountInterface {
public:
// By default, usage_state() is null until a measurement is made.
Resource();
- virtual ~Resource();
+ ~Resource() override;
void SetResourceListener(ResourceListener* listener);
@@ -56,12 +59,12 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const;
+ rtc::scoped_refptr<Resource> reason_resource) const;
virtual void OnAdaptationApplied(
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource);
+ rtc::scoped_refptr<Resource> reason_resource);
virtual std::string name() const = 0;
diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/resource_adaptation_processor.cc
index 79fb9da..0224ac3 100644
--- a/call/adaptation/resource_adaptation_processor.cc
+++ b/call/adaptation/resource_adaptation_processor.cc
@@ -10,6 +10,7 @@
#include "call/adaptation/resource_adaptation_processor.h"
+#include <algorithm>
#include <utility>
#include "absl/algorithm/container.h"
@@ -19,7 +20,9 @@
ResourceAdaptationProcessor::ResourceAdaptationProcessor(
VideoStreamInputStateProvider* input_state_provider,
VideoStreamEncoderObserver* encoder_stats_observer)
- : input_state_provider_(input_state_provider),
+ : sequence_checker_(),
+ is_resource_adaptation_enabled_(false),
+ input_state_provider_(input_state_provider),
encoder_stats_observer_(encoder_stats_observer),
resources_(),
degradation_preference_(DegradationPreference::DISABLED),
@@ -27,53 +30,115 @@
is_screenshare_(false),
stream_adapter_(std::make_unique<VideoStreamAdapter>()),
last_reported_source_restrictions_(),
- processing_in_progress_(false) {}
+ processing_in_progress_(false) {
+ sequence_checker_.Detach();
+}
-ResourceAdaptationProcessor::~ResourceAdaptationProcessor() = default;
+ResourceAdaptationProcessor::~ResourceAdaptationProcessor() {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ RTC_DCHECK(!is_resource_adaptation_enabled_);
+ RTC_DCHECK(adaptation_listeners_.empty())
+ << "There are listener(s) depending on a ResourceAdaptationProcessor "
+ << "being destroyed.";
+ RTC_DCHECK(resources_.empty())
+ << "There are resource(s) attached to a ResourceAdaptationProcessor "
+ << "being destroyed.";
+}
+
+void ResourceAdaptationProcessor::InitializeOnResourceAdaptationQueue() {
+ // Allows |sequence_checker_| to attach to the resource adaptation queue.
+ // The caller is responsible for ensuring that this is the current queue.
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+}
DegradationPreference ResourceAdaptationProcessor::degradation_preference()
const {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
return degradation_preference_;
}
DegradationPreference
ResourceAdaptationProcessor::effective_degradation_preference() const {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
return effective_degradation_preference_;
}
void ResourceAdaptationProcessor::StartResourceAdaptation() {
- for (auto* resource : resources_) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ if (is_resource_adaptation_enabled_)
+ return;
+ for (const auto& resource : resources_) {
resource->SetResourceListener(this);
}
+ is_resource_adaptation_enabled_ = true;
}
void ResourceAdaptationProcessor::StopResourceAdaptation() {
- for (auto* resource : resources_) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ if (!is_resource_adaptation_enabled_)
+ return;
+ for (const auto& resource : resources_) {
resource->SetResourceListener(nullptr);
}
+ is_resource_adaptation_enabled_ = false;
}
void ResourceAdaptationProcessor::AddAdaptationListener(
ResourceAdaptationProcessorListener* adaptation_listener) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ RTC_DCHECK(std::find(adaptation_listeners_.begin(),
+ adaptation_listeners_.end(),
+ adaptation_listener) == adaptation_listeners_.end());
adaptation_listeners_.push_back(adaptation_listener);
}
-void ResourceAdaptationProcessor::AddResource(Resource* resource) {
+void ResourceAdaptationProcessor::RemoveAdaptationListener(
+ ResourceAdaptationProcessorListener* adaptation_listener) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ auto it = std::find(adaptation_listeners_.begin(),
+ adaptation_listeners_.end(), adaptation_listener);
+ RTC_DCHECK(it != adaptation_listeners_.end());
+ adaptation_listeners_.erase(it);
+}
+
+void ResourceAdaptationProcessor::AddResource(
+ rtc::scoped_refptr<Resource> resource) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ // TODO(hbos): Allow adding resources while |is_resource_adaptation_enabled_|
+ // by registering as a listener of the resource on adding it.
+ RTC_DCHECK(!is_resource_adaptation_enabled_);
+ RTC_DCHECK(std::find(resources_.begin(), resources_.end(), resource) ==
+ resources_.end());
resources_.push_back(resource);
}
+void ResourceAdaptationProcessor::RemoveResource(
+ rtc::scoped_refptr<Resource> resource) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ // TODO(hbos): Allow removing resources while
+ // |is_resource_adaptation_enabled_| by unregistering as a listener of the
+ // resource on removing it.
+ RTC_DCHECK(!is_resource_adaptation_enabled_);
+ auto it = std::find(resources_.begin(), resources_.end(), resource);
+ RTC_DCHECK(it != resources_.end());
+ resources_.erase(it);
+}
+
void ResourceAdaptationProcessor::SetDegradationPreference(
DegradationPreference degradation_preference) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
degradation_preference_ = degradation_preference;
MaybeUpdateEffectiveDegradationPreference();
}
void ResourceAdaptationProcessor::SetIsScreenshare(bool is_screenshare) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
is_screenshare_ = is_screenshare;
MaybeUpdateEffectiveDegradationPreference();
}
void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
effective_degradation_preference_ =
(is_screenshare_ &&
degradation_preference_ == DegradationPreference::BALANCED)
@@ -84,13 +149,15 @@
}
void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
stream_adapter_->ClearRestrictions();
adaptations_counts_by_resource_.clear();
MaybeUpdateVideoSourceRestrictions(nullptr);
}
void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions(
- const Resource* reason) {
+ rtc::scoped_refptr<Resource> reason) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
VideoSourceRestrictions new_source_restrictions =
FilterRestrictionsByDegradationPreference(
stream_adapter_->source_restrictions(),
@@ -109,9 +176,10 @@
}
void ResourceAdaptationProcessor::OnResourceUsageStateMeasured(
- const Resource& resource) {
- RTC_DCHECK(resource.usage_state().has_value());
- switch (resource.usage_state().value()) {
+ rtc::scoped_refptr<Resource> resource) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ RTC_DCHECK(resource->usage_state().has_value());
+ switch (resource->usage_state().value()) {
case ResourceUsageState::kOveruse:
OnResourceOveruse(resource);
break;
@@ -123,6 +191,7 @@
bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation(
const VideoStreamInputState& input_state) const {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
return input_state.HasInputFrameSizeAndFramesPerSecond() &&
(effective_degradation_preference_ !=
DegradationPreference::MAINTAIN_RESOLUTION ||
@@ -130,14 +199,15 @@
}
void ResourceAdaptationProcessor::OnResourceUnderuse(
- const Resource& reason_resource) {
+ rtc::scoped_refptr<Resource> reason_resource) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
RTC_DCHECK(!processing_in_progress_);
processing_in_progress_ = true;
// Clear all usage states. In order to re-run adaptation logic, resources need
// to provide new resource usage measurements.
// TODO(hbos): Support not unconditionally clearing usage states by having the
// ResourceAdaptationProcessor check in on its resources at certain intervals.
- for (Resource* resource : resources_) {
+ for (const auto& resource : resources_) {
resource->ClearUsageState();
}
VideoStreamInputState input_state = input_state_provider_->InputState();
@@ -146,7 +216,7 @@
processing_in_progress_ = false;
return;
}
- if (!IsResourceAllowedToAdaptUp(&reason_resource)) {
+ if (!IsResourceAllowedToAdaptUp(reason_resource)) {
processing_in_progress_ = false;
return;
}
@@ -164,8 +234,8 @@
VideoSourceRestrictions restrictions_after =
stream_adapter_->PeekNextRestrictions(adaptation);
if (!absl::c_all_of(resources_, [&input_state, &restrictions_before,
- &restrictions_after,
- &reason_resource](const Resource* resource) {
+ &restrictions_after, &reason_resource](
+ rtc::scoped_refptr<Resource> resource) {
return resource->IsAdaptationUpAllowed(input_state, restrictions_before,
restrictions_after,
reason_resource);
@@ -175,25 +245,26 @@
}
// Apply adaptation.
stream_adapter_->ApplyAdaptation(adaptation);
- for (Resource* resource : resources_) {
+ for (const auto& resource : resources_) {
resource->OnAdaptationApplied(input_state, restrictions_before,
restrictions_after, reason_resource);
}
// Update VideoSourceRestrictions based on adaptation. This also informs the
// |adaptation_listeners_|.
- MaybeUpdateVideoSourceRestrictions(&reason_resource);
+ MaybeUpdateVideoSourceRestrictions(reason_resource);
processing_in_progress_ = false;
}
void ResourceAdaptationProcessor::OnResourceOveruse(
- const Resource& reason_resource) {
+ rtc::scoped_refptr<Resource> reason_resource) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
RTC_DCHECK(!processing_in_progress_);
processing_in_progress_ = true;
// Clear all usage states. In order to re-run adaptation logic, resources need
// to provide new resource usage measurements.
// TODO(hbos): Support not unconditionally clearing usage states by having the
// ResourceAdaptationProcessor check in on its resources at certain intervals.
- for (Resource* resource : resources_) {
+ for (const auto& resource : resources_) {
resource->ClearUsageState();
}
VideoStreamInputState input_state = input_state_provider_->InputState();
@@ -223,18 +294,19 @@
VideoSourceRestrictions restrictions_after =
stream_adapter_->PeekNextRestrictions(adaptation);
stream_adapter_->ApplyAdaptation(adaptation);
- for (Resource* resource : resources_) {
+ for (const auto& resource : resources_) {
resource->OnAdaptationApplied(input_state, restrictions_before,
restrictions_after, reason_resource);
}
// Update VideoSourceRestrictions based on adaptation. This also informs the
// |adaptation_listeners_|.
- MaybeUpdateVideoSourceRestrictions(&reason_resource);
+ MaybeUpdateVideoSourceRestrictions(reason_resource);
processing_in_progress_ = false;
}
void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize(
- const Resource& reason_resource) {
+ rtc::scoped_refptr<Resource> reason_resource) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
VideoAdaptationCounters counters_before =
stream_adapter_->adaptation_counters();
OnResourceOveruse(reason_resource);
@@ -252,7 +324,8 @@
}
void ResourceAdaptationProcessor::UpdateResourceDegradationCounts(
- const Resource* resource) {
+ rtc::scoped_refptr<Resource> resource) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
RTC_DCHECK(resource);
int delta = stream_adapter_->adaptation_counters().Total();
for (const auto& adaptations : adaptations_counts_by_resource_) {
@@ -265,7 +338,8 @@
}
bool ResourceAdaptationProcessor::IsResourceAllowedToAdaptUp(
- const Resource* resource) const {
+ rtc::scoped_refptr<Resource> resource) const {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
RTC_DCHECK(resource);
const auto& adaptations = adaptations_counts_by_resource_.find(resource);
return adaptations != adaptations_counts_by_resource_.end() &&
diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h
index de38751..cf1e187 100644
--- a/call/adaptation/resource_adaptation_processor.h
+++ b/call/adaptation/resource_adaptation_processor.h
@@ -17,6 +17,7 @@
#include "absl/types/optional.h"
#include "api/rtp_parameters.h"
+#include "api/scoped_refptr.h"
#include "api/video/video_frame.h"
#include "api/video/video_stream_encoder_observer.h"
#include "call/adaptation/resource.h"
@@ -25,9 +26,26 @@
#include "call/adaptation/video_stream_adapter.h"
#include "call/adaptation/video_stream_input_state.h"
#include "call/adaptation/video_stream_input_state_provider.h"
+#include "rtc_base/synchronization/sequence_checker.h"
namespace webrtc {
+// The Resource Adaptation Processor is responsible for reacting to resource
+// usage measurements (e.g. overusing or underusing CPU). When a resource is
+// overused the Processor is responsible for performing mitigations in order to
+// consume less resources.
+//
+// Today we have one Processor per VideoStreamEncoder and the Processor is only
+// capable of restricting resolution or frame rate of the encoded stream. In the
+// future we should have a single Processor responsible for all encoded streams,
+// and it should be capable of reconfiguring other things than just
+// VideoSourceRestrictions (e.g. reduce render frame rate).
+// See Resource-Adaptation hotlist:
+// https://bugs.chromium.org/u/590058293/hotlists/Resource-Adaptation
+//
+// The ResourceAdaptationProcessor is single-threaded. It may be constructed on
+// any thread but MUST subsequently be used and destroyed on a single sequence,
+// i.e. the "resource adaptation task queue".
class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
public ResourceListener {
public:
@@ -36,6 +54,8 @@
VideoStreamEncoderObserver* encoder_stats_observer);
~ResourceAdaptationProcessor() override;
+ void InitializeOnResourceAdaptationQueue() override;
+
// ResourceAdaptationProcessorInterface implementation.
DegradationPreference degradation_preference() const override;
DegradationPreference effective_degradation_preference() const override;
@@ -44,7 +64,10 @@
void StopResourceAdaptation() override;
void AddAdaptationListener(
ResourceAdaptationProcessorListener* adaptation_listener) override;
- void AddResource(Resource* resource) override;
+ void RemoveAdaptationListener(
+ ResourceAdaptationProcessorListener* adaptation_listener) override;
+ void AddResource(rtc::scoped_refptr<Resource> resource) override;
+ void RemoveResource(rtc::scoped_refptr<Resource> resource) override;
void SetDegradationPreference(
DegradationPreference degradation_preference) override;
@@ -53,13 +76,14 @@
// ResourceListener implementation.
// Triggers OnResourceUnderuse() or OnResourceOveruse().
- void OnResourceUsageStateMeasured(const Resource& resource) override;
+ void OnResourceUsageStateMeasured(
+ rtc::scoped_refptr<Resource> resource) override;
// May trigger 1-2 adaptations. It is meant to reduce resolution but this is
// not guaranteed. It may adapt frame rate, which does not address the issue.
// TODO(hbos): Can we get rid of this?
void TriggerAdaptationDueToFrameDroppedDueToSize(
- const Resource& reason_resource) override;
+ rtc::scoped_refptr<Resource> reason_resource) override;
private:
bool HasSufficientInputForAdaptation(
@@ -68,18 +92,18 @@
// Performs the adaptation by getting the next target, applying it and
// informing listeners of the new VideoSourceRestriction and adaptation
// counters.
- void OnResourceUnderuse(const Resource& reason_resource);
- void OnResourceOveruse(const Resource& reason_resource);
+ void OnResourceUnderuse(rtc::scoped_refptr<Resource> reason_resource);
+ void OnResourceOveruse(rtc::scoped_refptr<Resource> reason_resource);
// Needs to be invoked any time |degradation_preference_| or |is_screenshare_|
// changes to ensure |effective_degradation_preference_| is up-to-date.
void MaybeUpdateEffectiveDegradationPreference();
// If the filtered source restrictions are different than
// |last_reported_source_restrictions_|, inform the listeners.
- void MaybeUpdateVideoSourceRestrictions(const Resource* reason);
+ void MaybeUpdateVideoSourceRestrictions(rtc::scoped_refptr<Resource> reason);
// Updates the number of times the resource has degraded based on the latest
// degradation applied.
- void UpdateResourceDegradationCounts(const Resource* resource);
+ void UpdateResourceDegradationCounts(rtc::scoped_refptr<Resource> resource);
// Returns true if a Resource has been overused in the pass and is responsible
// for creating a VideoSourceRestriction. The current algorithm counts the
// number of times the resource caused an adaptation and allows adapting up
@@ -87,21 +111,33 @@
// traditionally been handled.
// TODO(crbug.com/webrtc/11553) Change this algorithm to look at the resources
// restrictions rather than just the counters.
- bool IsResourceAllowedToAdaptUp(const Resource* resource) const;
+ bool IsResourceAllowedToAdaptUp(rtc::scoped_refptr<Resource> resource) const;
+ webrtc::SequenceChecker sequence_checker_;
+ bool is_resource_adaptation_enabled_ RTC_GUARDED_BY(sequence_checker_);
// Input and output.
- VideoStreamInputStateProvider* const input_state_provider_;
- VideoStreamEncoderObserver* const encoder_stats_observer_;
- std::vector<ResourceAdaptationProcessorListener*> adaptation_listeners_;
- std::vector<Resource*> resources_;
- std::map<const Resource*, int> adaptations_counts_by_resource_;
+ VideoStreamInputStateProvider* const input_state_provider_
+ RTC_GUARDED_BY(sequence_checker_);
+ VideoStreamEncoderObserver* const encoder_stats_observer_
+ RTC_GUARDED_BY(sequence_checker_);
+ std::vector<ResourceAdaptationProcessorListener*> adaptation_listeners_
+ RTC_GUARDED_BY(sequence_checker_);
+ std::vector<rtc::scoped_refptr<Resource>> resources_
+ RTC_GUARDED_BY(sequence_checker_);
+ // Purely used for statistics, does not ensure mapped resources stay alive.
+ std::map<const Resource*, int> adaptations_counts_by_resource_
+ RTC_GUARDED_BY(sequence_checker_);
// Adaptation strategy settings.
- DegradationPreference degradation_preference_;
- DegradationPreference effective_degradation_preference_;
- bool is_screenshare_;
+ DegradationPreference degradation_preference_
+ RTC_GUARDED_BY(sequence_checker_);
+ DegradationPreference effective_degradation_preference_
+ RTC_GUARDED_BY(sequence_checker_);
+ bool is_screenshare_ RTC_GUARDED_BY(sequence_checker_);
// Responsible for generating and applying possible adaptations.
- const std::unique_ptr<VideoStreamAdapter> stream_adapter_;
- VideoSourceRestrictions last_reported_source_restrictions_;
+ const std::unique_ptr<VideoStreamAdapter> stream_adapter_
+ RTC_GUARDED_BY(sequence_checker_);
+ VideoSourceRestrictions last_reported_source_restrictions_
+ RTC_GUARDED_BY(sequence_checker_);
// Prevents recursion.
//
// This is used to prevent triggering resource adaptation in the process of
@@ -113,7 +149,7 @@
// Resource::OnAdaptationApplied() ->
// Resource::OnResourceUsageStateMeasured() ->
// ResourceAdaptationProcessor::OnResourceOveruse() // Boom, not allowed.
- bool processing_in_progress_;
+ bool processing_in_progress_ RTC_GUARDED_BY(sequence_checker_);
};
} // namespace webrtc
diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h
index 6984273..d6295c4 100644
--- a/call/adaptation/resource_adaptation_processor_interface.h
+++ b/call/adaptation/resource_adaptation_processor_interface.h
@@ -13,11 +13,13 @@
#include "absl/types/optional.h"
#include "api/rtp_parameters.h"
+#include "api/scoped_refptr.h"
#include "api/video/video_adaptation_counters.h"
#include "api/video/video_frame.h"
#include "call/adaptation/encoder_settings.h"
#include "call/adaptation/resource.h"
#include "call/adaptation/video_source_restrictions.h"
+#include "rtc_base/task_queue.h"
namespace webrtc {
@@ -33,16 +35,19 @@
virtual void OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
- const Resource* reason) = 0;
+ rtc::scoped_refptr<Resource> reason) = 0;
};
-// Responsible for reconfiguring encoded streams based on resource consumption,
-// such as scaling down resolution or frame rate when CPU is overused. This
-// interface is meant to be injectable into VideoStreamEncoder.
+// The Resource Adaptation Processor is responsible for reacting to resource
+// usage measurements (e.g. overusing or underusing CPU). When a resource is
+// overused the Processor is responsible for performing mitigations in order to
+// consume less resources.
class ResourceAdaptationProcessorInterface {
public:
virtual ~ResourceAdaptationProcessorInterface();
+ virtual void InitializeOnResourceAdaptationQueue() = 0;
+
virtual DegradationPreference degradation_preference() const = 0;
// Reinterprets "balanced + screenshare" as "maintain-resolution".
// TODO(hbos): Don't do this. This is not what "balanced" means. If the
@@ -60,7 +65,10 @@
virtual void StopResourceAdaptation() = 0;
virtual void AddAdaptationListener(
ResourceAdaptationProcessorListener* adaptation_listener) = 0;
- virtual void AddResource(Resource* resource) = 0;
+ virtual void RemoveAdaptationListener(
+ ResourceAdaptationProcessorListener* adaptation_listener) = 0;
+ virtual void AddResource(rtc::scoped_refptr<Resource> resource) = 0;
+ virtual void RemoveResource(rtc::scoped_refptr<Resource> resource) = 0;
virtual void SetDegradationPreference(
DegradationPreference degradation_preference) = 0;
@@ -74,7 +82,7 @@
// reasons. Can we replace this by something which actually satisfies the
// resolution constraints or get rid of it altogether?
virtual void TriggerAdaptationDueToFrameDroppedDueToSize(
- const Resource& reason_resource) = 0;
+ rtc::scoped_refptr<Resource> reason_resource) = 0;
};
} // namespace webrtc
diff --git a/call/adaptation/resource_adaptation_processor_unittest.cc b/call/adaptation/resource_adaptation_processor_unittest.cc
index 7e7fe59..68dc4ba 100644
--- a/call/adaptation/resource_adaptation_processor_unittest.cc
+++ b/call/adaptation/resource_adaptation_processor_unittest.cc
@@ -10,6 +10,7 @@
#include "call/adaptation/resource_adaptation_processor.h"
+#include "api/scoped_refptr.h"
#include "api/video/video_adaptation_counters.h"
#include "call/adaptation/resource.h"
#include "call/adaptation/resource_adaptation_processor_interface.h"
@@ -43,13 +44,13 @@
const VideoAdaptationCounters& adaptation_counters() const {
return adaptation_counters_;
}
- const Resource* reason() const { return reason_; }
+ rtc::scoped_refptr<Resource> reason() const { return reason_; }
// ResourceAdaptationProcessorListener implementation.
void OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
- const Resource* reason) override {
+ rtc::scoped_refptr<Resource> reason) override {
++restrictions_updated_count_;
restrictions_ = restrictions;
adaptation_counters_ = adaptation_counters;
@@ -60,7 +61,7 @@
size_t restrictions_updated_count_;
VideoSourceRestrictions restrictions_;
VideoAdaptationCounters adaptation_counters_;
- const Resource* reason_;
+ rtc::scoped_refptr<Resource> reason_;
};
class ResourceAdaptationProcessorTest : public ::testing::Test {
@@ -68,16 +69,20 @@
ResourceAdaptationProcessorTest()
: frame_rate_provider_(),
input_state_provider_(&frame_rate_provider_),
- resource_("FakeResource"),
- other_resource_("OtherFakeResource"),
+ resource_(new FakeResource("FakeResource")),
+ other_resource_(new FakeResource("OtherFakeResource")),
processor_(&input_state_provider_,
/*encoder_stats_observer=*/&frame_rate_provider_) {
+ processor_.InitializeOnResourceAdaptationQueue();
processor_.AddAdaptationListener(&processor_listener_);
- processor_.AddResource(&resource_);
- processor_.AddResource(&other_resource_);
+ processor_.AddResource(resource_);
+ processor_.AddResource(other_resource_);
}
~ResourceAdaptationProcessorTest() override {
processor_.StopResourceAdaptation();
+ processor_.RemoveResource(resource_);
+ processor_.RemoveResource(other_resource_);
+ processor_.RemoveAdaptationListener(&processor_listener_);
}
void SetInputStates(bool has_input, int fps, int frame_size) {
@@ -97,8 +102,8 @@
protected:
FakeFrameRateProvider frame_rate_provider_;
VideoStreamInputStateProvider input_state_provider_;
- FakeResource resource_;
- FakeResource other_resource_;
+ rtc::scoped_refptr<FakeResource> resource_;
+ rtc::scoped_refptr<FakeResource> other_resource_;
ResourceAdaptationProcessor processor_;
ResourceAdaptationProcessorListenerForTesting processor_listener_;
};
@@ -113,7 +118,7 @@
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
processor_.StartResourceAdaptation();
// Adaptation does not happen when disabled.
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(0u, processor_listener_.restrictions_updated_count());
}
@@ -124,11 +129,11 @@
// Adaptation does not happen if input is insufficient.
// When frame size is missing (OnFrameSizeObserved not called yet).
input_state_provider_.OnHasInputChanged(true);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(0u, processor_listener_.restrictions_updated_count());
// When "has input" is missing.
SetInputStates(false, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(0u, processor_listener_.restrictions_updated_count());
// Note: frame rate cannot be missing, if unset it is 0.
}
@@ -143,7 +148,7 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
EXPECT_TRUE(
processor_listener_.restrictions().max_pixels_per_frame().has_value());
@@ -155,7 +160,7 @@
DegradationPreference::MAINTAIN_RESOLUTION);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
EXPECT_TRUE(processor_listener_.restrictions().max_frame_rate().has_value());
}
@@ -170,7 +175,7 @@
// VideoStreamAdapter and default input states. This test requires it to be
// achieved within 4 adaptations.
for (size_t i = 0; i < 4; ++i) {
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(i + 1, processor_listener_.restrictions_updated_count());
RestrictSource(processor_listener_.restrictions());
}
@@ -184,11 +189,11 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
// If we don't restrict the source then adaptation will not happen again due
// to "awaiting previous adaptation". This prevents "double-adapt".
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
}
@@ -197,7 +202,7 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(0u, processor_listener_.restrictions_updated_count());
}
@@ -206,10 +211,10 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
RestrictSource(processor_listener_.restrictions());
- resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(2u, processor_listener_.restrictions_updated_count());
EXPECT_EQ(VideoSourceRestrictions(), processor_listener_.restrictions());
}
@@ -220,12 +225,12 @@
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
// Adapt down so that we can adapt up.
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
RestrictSource(processor_listener_.restrictions());
// Adapting up is prevented.
- resource_.set_is_adaptation_up_allowed(false);
- resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ resource_->set_is_adaptation_up_allowed(false);
+ resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
}
@@ -235,12 +240,12 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
RestrictSource(processor_listener_.restrictions());
// Other resource signals under-use
- other_resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ other_resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
}
@@ -250,18 +255,18 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
processor_.ResetVideoSourceRestrictions();
EXPECT_EQ(0, processor_listener_.adaptation_counters().Total());
- other_resource_.set_usage_state(ResourceUsageState::kOveruse);
+ other_resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
// resource_ did not overuse after we reset the restrictions, so adapt up
// should be disallowed.
- resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(1, processor_listener_.adaptation_counters().Total());
}
@@ -271,28 +276,28 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
- other_resource_.set_usage_state(ResourceUsageState::kOveruse);
+ other_resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(2, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
- other_resource_.set_usage_state(ResourceUsageState::kOveruse);
+ other_resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(3, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
- resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(2, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
// Does not trigger adaptation since resource has no adaptations left.
- resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(2, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
- other_resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ other_resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(1, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
- other_resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ other_resource_->set_usage_state(ResourceUsageState::kUnderuse);
EXPECT_EQ(0, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
}
@@ -302,8 +307,8 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
- EXPECT_EQ(1u, resource_.num_adaptations_applied());
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
+ EXPECT_EQ(1u, resource_->num_adaptations_applied());
}
TEST_F(ResourceAdaptationProcessorTest, AdaptingClearsResourceUsageState) {
@@ -311,18 +316,18 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(1u, processor_listener_.restrictions_updated_count());
- EXPECT_FALSE(resource_.usage_state().has_value());
+ EXPECT_FALSE(resource_->usage_state().has_value());
}
TEST_F(ResourceAdaptationProcessorTest,
FailingAdaptingAlsoClearsResourceUsageState) {
processor_.SetDegradationPreference(DegradationPreference::DISABLED);
processor_.StartResourceAdaptation();
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
EXPECT_EQ(0u, processor_listener_.restrictions_updated_count());
- EXPECT_FALSE(resource_.usage_state().has_value());
+ EXPECT_FALSE(resource_->usage_state().has_value());
}
TEST_F(ResourceAdaptationProcessorTest,
@@ -331,17 +336,17 @@
DegradationPreference::MAINTAIN_FRAMERATE);
processor_.StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
- other_resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ other_resource_->set_usage_state(ResourceUsageState::kUnderuse);
// Does not trigger adapataion because there's no restriction.
EXPECT_EQ(0, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
- resource_.set_usage_state(ResourceUsageState::kOveruse);
+ resource_->set_usage_state(ResourceUsageState::kOveruse);
// Adapts down even if other resource asked for adapting up.
EXPECT_EQ(1, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
- other_resource_.set_usage_state(ResourceUsageState::kUnderuse);
+ other_resource_->set_usage_state(ResourceUsageState::kUnderuse);
// Doesn't adapt up because adaptation is due to another resource.
EXPECT_EQ(1, processor_listener_.adaptation_counters().Total());
RestrictSource(processor_listener_.restrictions());
diff --git a/call/adaptation/resource_unittest.cc b/call/adaptation/resource_unittest.cc
index d864005..50a6220 100644
--- a/call/adaptation/resource_unittest.cc
+++ b/call/adaptation/resource_unittest.cc
@@ -10,6 +10,7 @@
#include "call/adaptation/resource.h"
+#include "api/scoped_refptr.h"
#include "call/adaptation/test/fake_resource.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -21,29 +22,33 @@
class MockResourceListener : public ResourceListener {
public:
- MOCK_METHOD(void, OnResourceUsageStateMeasured, (const Resource& resource));
+ MOCK_METHOD(void,
+ OnResourceUsageStateMeasured,
+ (rtc::scoped_refptr<Resource> resource));
};
TEST(ResourceTest, RegisteringListenerReceivesCallbacks) {
StrictMock<MockResourceListener> resource_listener;
- FakeResource fake_resource("FakeResource");
- fake_resource.SetResourceListener(&resource_listener);
+ rtc::scoped_refptr<FakeResource> fake_resource(
+ new FakeResource("FakeResource"));
+ fake_resource->SetResourceListener(&resource_listener);
EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_))
.Times(1)
- .WillOnce([](const Resource& resource) {
- EXPECT_EQ(ResourceUsageState::kOveruse, resource.usage_state());
+ .WillOnce([](rtc::scoped_refptr<Resource> resource) {
+ EXPECT_EQ(ResourceUsageState::kOveruse, resource->usage_state());
});
- fake_resource.set_usage_state(ResourceUsageState::kOveruse);
- fake_resource.SetResourceListener(nullptr);
+ fake_resource->set_usage_state(ResourceUsageState::kOveruse);
+ fake_resource->SetResourceListener(nullptr);
}
TEST(ResourceTest, UnregisteringListenerStopsCallbacks) {
StrictMock<MockResourceListener> resource_listener;
- FakeResource fake_resource("FakeResource");
- fake_resource.SetResourceListener(&resource_listener);
- fake_resource.SetResourceListener(nullptr);
+ rtc::scoped_refptr<FakeResource> fake_resource(
+ new FakeResource("FakeResource"));
+ fake_resource->SetResourceListener(&resource_listener);
+ fake_resource->SetResourceListener(nullptr);
EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)).Times(0);
- fake_resource.set_usage_state(ResourceUsageState::kOveruse);
+ fake_resource->set_usage_state(ResourceUsageState::kOveruse);
}
} // namespace webrtc
diff --git a/call/adaptation/test/fake_resource.cc b/call/adaptation/test/fake_resource.cc
index bd7ad54..4c0a129 100644
--- a/call/adaptation/test/fake_resource.cc
+++ b/call/adaptation/test/fake_resource.cc
@@ -15,7 +15,7 @@
namespace webrtc {
FakeResource::FakeResource(std::string name)
- : Resource(),
+ : rtc::RefCountedObject<Resource>(),
name_(std::move(name)),
is_adaptation_up_allowed_(true),
num_adaptations_applied_(0) {}
@@ -38,7 +38,7 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const {
+ rtc::scoped_refptr<Resource> reason_resource) const {
return is_adaptation_up_allowed_;
}
@@ -46,7 +46,7 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) {
+ rtc::scoped_refptr<Resource> reason_resource) {
++num_adaptations_applied_;
}
diff --git a/call/adaptation/test/fake_resource.h b/call/adaptation/test/fake_resource.h
index 0d9b1f4..beaca54 100644
--- a/call/adaptation/test/fake_resource.h
+++ b/call/adaptation/test/fake_resource.h
@@ -14,11 +14,12 @@
#include <string>
#include "call/adaptation/resource.h"
+#include "rtc_base/ref_counted_object.h"
namespace webrtc {
// Fake resource used for testing.
-class FakeResource : public Resource {
+class FakeResource : public rtc::RefCountedObject<Resource> {
public:
explicit FakeResource(std::string name);
~FakeResource() override;
@@ -29,14 +30,16 @@
// Resource implementation.
std::string name() const override { return name_; }
- bool IsAdaptationUpAllowed(const VideoStreamInputState& input_state,
- const VideoSourceRestrictions& restrictions_before,
- const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const override;
- void OnAdaptationApplied(const VideoStreamInputState& input_state,
- const VideoSourceRestrictions& restrictions_before,
- const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) override;
+ bool IsAdaptationUpAllowed(
+ const VideoStreamInputState& input_state,
+ const VideoSourceRestrictions& restrictions_before,
+ const VideoSourceRestrictions& restrictions_after,
+ rtc::scoped_refptr<Resource> reason_resource) const override;
+ void OnAdaptationApplied(
+ const VideoStreamInputState& input_state,
+ const VideoSourceRestrictions& restrictions_before,
+ const VideoSourceRestrictions& restrictions_after,
+ rtc::scoped_refptr<Resource> reason_resource) override;
private:
const std::string name_;
diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn
index 1224586..3269d89 100644
--- a/video/adaptation/BUILD.gn
+++ b/video/adaptation/BUILD.gn
@@ -22,6 +22,7 @@
deps = [
"../../api:rtp_parameters",
+ "../../api:scoped_refptr",
"../../api/task_queue:task_queue",
"../../api/video:video_adaptation",
"../../api/video:video_frame",
@@ -35,6 +36,7 @@
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_event",
"../../rtc_base:rtc_numerics",
+ "../../rtc_base:rtc_task_queue",
"../../rtc_base:timeutils",
"../../rtc_base/experiments:balanced_degradation_settings",
"../../rtc_base/experiments:field_trial_parser",
@@ -61,6 +63,7 @@
]
deps = [
":video_adaptation",
+ "../../api:scoped_refptr",
"../../api/video:encoded_image",
"../../api/video:video_adaptation",
"../../api/video:video_frame_i420",
diff --git a/video/adaptation/encode_usage_resource.cc b/video/adaptation/encode_usage_resource.cc
index 7a42878..6e2827a 100644
--- a/video/adaptation/encode_usage_resource.cc
+++ b/video/adaptation/encode_usage_resource.cc
@@ -20,13 +20,27 @@
EncodeUsageResource::EncodeUsageResource(
std::unique_ptr<OveruseFrameDetector> overuse_detector)
- : overuse_detector_(std::move(overuse_detector)),
+ : rtc::RefCountedObject<Resource>(),
+ encoder_queue_(nullptr),
+ overuse_detector_(std::move(overuse_detector)),
is_started_(false),
target_frame_rate_(absl::nullopt) {
RTC_DCHECK(overuse_detector_);
}
+void EncodeUsageResource::Initialize(rtc::TaskQueue* encoder_queue) {
+ RTC_DCHECK(!encoder_queue_);
+ RTC_DCHECK(encoder_queue);
+ encoder_queue_ = encoder_queue;
+}
+
+bool EncodeUsageResource::is_started() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ return is_started_;
+}
+
void EncodeUsageResource::StartCheckForOveruse(CpuOveruseOptions options) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(!is_started_);
overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(),
std::move(options), this);
@@ -35,12 +49,14 @@
}
void EncodeUsageResource::StopCheckForOveruse() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
overuse_detector_->StopCheckForOveruse();
is_started_ = false;
}
void EncodeUsageResource::SetTargetFrameRate(
absl::optional<double> target_frame_rate) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
if (target_frame_rate == target_frame_rate_)
return;
target_frame_rate_ = target_frame_rate;
@@ -50,6 +66,7 @@
void EncodeUsageResource::OnEncodeStarted(const VideoFrame& cropped_frame,
int64_t time_when_first_seen_us) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
// TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g.
// "OnEncodeStarted"?) or revise usage.
overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us);
@@ -60,6 +77,7 @@
int64_t time_sent_in_us,
int64_t capture_time_us,
absl::optional<int> encode_duration_us) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
// TODO(hbos): Rename FrameSent() to something more appropriate (e.g.
// "OnEncodeCompleted"?).
overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us,
@@ -67,14 +85,21 @@
}
void EncodeUsageResource::AdaptUp() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // PostTask the resource usage measurements.
OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse);
}
void EncodeUsageResource::AdaptDown() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // PostTask the resource usage measurements.
OnResourceUsageStateMeasured(ResourceUsageState::kOveruse);
}
int EncodeUsageResource::TargetFrameRateAsInt() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
return target_frame_rate_.has_value()
? static_cast<int>(target_frame_rate_.value())
: std::numeric_limits<int>::max();
diff --git a/video/adaptation/encode_usage_resource.h b/video/adaptation/encode_usage_resource.h
index 7147569..2ade816 100644
--- a/video/adaptation/encode_usage_resource.h
+++ b/video/adaptation/encode_usage_resource.h
@@ -17,6 +17,8 @@
#include "absl/types/optional.h"
#include "api/video/video_adaptation_reason.h"
#include "call/adaptation/resource.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/task_queue.h"
#include "video/adaptation/overuse_frame_detector.h"
namespace webrtc {
@@ -26,13 +28,17 @@
// indirectly by usage in the ResourceAdaptationProcessor (which is only tested
// because of its usage in VideoStreamEncoder); all tests are currently in
// video_stream_encoder_unittest.cc.
-class EncodeUsageResource : public Resource,
+class EncodeUsageResource : public rtc::RefCountedObject<Resource>,
public OveruseFrameDetectorObserverInterface {
public:
explicit EncodeUsageResource(
std::unique_ptr<OveruseFrameDetector> overuse_detector);
- bool is_started() const { return is_started_; }
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // pass it in here.
+ void Initialize(rtc::TaskQueue* encoder_queue);
+
+ bool is_started() const;
void StartCheckForOveruse(CpuOveruseOptions options);
void StopCheckForOveruse();
@@ -54,9 +60,11 @@
private:
int TargetFrameRateAsInt();
- const std::unique_ptr<OveruseFrameDetector> overuse_detector_;
- bool is_started_;
- absl::optional<double> target_frame_rate_;
+ rtc::TaskQueue* encoder_queue_;
+ const std::unique_ptr<OveruseFrameDetector> overuse_detector_
+ RTC_GUARDED_BY(encoder_queue_);
+ bool is_started_ RTC_GUARDED_BY(encoder_queue_);
+ absl::optional<double> target_frame_rate_ RTC_GUARDED_BY(encoder_queue_);
};
} // namespace webrtc
diff --git a/video/adaptation/quality_scaler_resource.cc b/video/adaptation/quality_scaler_resource.cc
index ca317e5..9fcc58e 100644
--- a/video/adaptation/quality_scaler_resource.cc
+++ b/video/adaptation/quality_scaler_resource.cc
@@ -16,40 +16,61 @@
namespace webrtc {
-QualityScalerResource::QualityScalerResource(
- ResourceAdaptationProcessorInterface* adaptation_processor)
- : adaptation_processor_(adaptation_processor),
+QualityScalerResource::QualityScalerResource()
+ : rtc::RefCountedObject<Resource>(),
+ encoder_queue_(nullptr),
+ adaptation_processor_(nullptr),
quality_scaler_(nullptr),
pending_qp_usage_callback_(nullptr) {}
+void QualityScalerResource::Initialize(rtc::TaskQueue* encoder_queue) {
+ RTC_DCHECK(!encoder_queue_);
+ RTC_DCHECK(encoder_queue);
+ encoder_queue_ = encoder_queue;
+}
+
+QualityScalerResource::~QualityScalerResource() {}
+
+void QualityScalerResource::SetAdaptationProcessor(
+ ResourceAdaptationProcessorInterface* adaptation_processor) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ adaptation_processor_ = adaptation_processor;
+}
+
bool QualityScalerResource::is_started() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
return quality_scaler_.get();
}
void QualityScalerResource::StartCheckForOveruse(
VideoEncoder::QpThresholds qp_thresholds) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(!is_started());
quality_scaler_ =
std::make_unique<QualityScaler>(this, std::move(qp_thresholds));
}
void QualityScalerResource::StopCheckForOveruse() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
quality_scaler_.reset();
}
void QualityScalerResource::SetQpThresholds(
VideoEncoder::QpThresholds qp_thresholds) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(is_started());
quality_scaler_->SetQpThresholds(std::move(qp_thresholds));
}
bool QualityScalerResource::QpFastFilterLow() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(is_started());
return quality_scaler_->QpFastFilterLow();
}
void QualityScalerResource::OnEncodeCompleted(const EncodedImage& encoded_image,
int64_t time_sent_in_us) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
if (quality_scaler_ && encoded_image.qp_ >= 0) {
quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us);
} else if (!quality_scaler_) {
@@ -64,6 +85,7 @@
void QualityScalerResource::OnFrameDropped(
EncodedImageCallback::DropReason reason) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
if (!quality_scaler_)
return;
switch (reason) {
@@ -78,7 +100,10 @@
void QualityScalerResource::OnReportQpUsageHigh(
rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(!pending_qp_usage_callback_);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // PostTask the resource usage measurements.
pending_qp_usage_callback_ = std::move(callback);
// If this triggers adaptation, OnAdaptationApplied() is called by the
// processor where we determine if QP should be cleared and we invoke and null
@@ -94,7 +119,10 @@
void QualityScalerResource::OnReportQpUsageLow(
rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(!pending_qp_usage_callback_);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // PostTask the resource usage measurements.
OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse);
callback->OnQpUsageHandled(true);
}
@@ -103,7 +131,10 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) {
+ rtc::scoped_refptr<Resource> reason_resource) {
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // ensure that this is running on it instead.
+ RTC_DCHECK_RUN_ON(encoder_queue_);
// We only clear QP samples on adaptations triggered by the QualityScaler.
if (!pending_qp_usage_callback_)
return;
@@ -119,7 +150,8 @@
// interval whose delay is calculated based on events such as these. Now there
// is much dependency on a specific OnReportQpUsageHigh() event and "balanced"
// but adaptations happening might not align with QualityScaler's CheckQpTask.
- if (adaptation_processor_->effective_degradation_preference() ==
+ if (adaptation_processor_ &&
+ adaptation_processor_->effective_degradation_preference() ==
DegradationPreference::BALANCED &&
DidDecreaseFrameRate(restrictions_before, restrictions_after)) {
absl::optional<int> min_diff = BalancedDegradationSettings().MinFpsDiff(
diff --git a/video/adaptation/quality_scaler_resource.h b/video/adaptation/quality_scaler_resource.h
index 30796c3..6cec79c 100644
--- a/video/adaptation/quality_scaler_resource.h
+++ b/video/adaptation/quality_scaler_resource.h
@@ -19,6 +19,8 @@
#include "call/adaptation/resource.h"
#include "call/adaptation/resource_adaptation_processor_interface.h"
#include "modules/video_coding/utility/quality_scaler.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/task_queue.h"
namespace webrtc {
@@ -27,10 +29,16 @@
// indirectly by usage in the ResourceAdaptationProcessor (which is only tested
// because of its usage in VideoStreamEncoder); all tests are currently in
// video_stream_encoder_unittest.cc.
-class QualityScalerResource : public Resource,
+class QualityScalerResource : public rtc::RefCountedObject<Resource>,
public QualityScalerQpUsageHandlerInterface {
public:
- explicit QualityScalerResource(
+ QualityScalerResource();
+ ~QualityScalerResource() override;
+
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // pass it in here.
+ void Initialize(rtc::TaskQueue* encoder_queue);
+ void SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor);
bool is_started() const;
@@ -55,16 +63,21 @@
std::string name() const override { return "QualityScalerResource"; }
// Resource implementation.
- void OnAdaptationApplied(const VideoStreamInputState& input_state,
- const VideoSourceRestrictions& restrictions_before,
- const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) override;
+ void OnAdaptationApplied(
+ const VideoStreamInputState& input_state,
+ const VideoSourceRestrictions& restrictions_before,
+ const VideoSourceRestrictions& restrictions_after,
+ rtc::scoped_refptr<Resource> reason_resource) override;
private:
- ResourceAdaptationProcessorInterface* const adaptation_processor_;
- std::unique_ptr<QualityScaler> quality_scaler_;
+ rtc::TaskQueue* encoder_queue_;
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // guard the processor by it instead.
+ ResourceAdaptationProcessorInterface* adaptation_processor_
+ RTC_GUARDED_BY(encoder_queue_);
+ std::unique_ptr<QualityScaler> quality_scaler_ RTC_GUARDED_BY(encoder_queue_);
rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
- pending_qp_usage_callback_;
+ pending_qp_usage_callback_ RTC_GUARDED_BY(encoder_queue_);
};
} // namespace webrtc
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc
index b8179d0..f5c812a 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.cc
+++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -68,7 +68,8 @@
class VideoStreamEncoderResourceManager::InitialFrameDropper {
public:
- explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource)
+ explicit InitialFrameDropper(
+ rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource)
: quality_scaler_resource_(quality_scaler_resource),
quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()),
has_seen_first_bwe_drop_(false),
@@ -128,7 +129,7 @@
// achieve desired bitrate.
static const int kMaxInitialFramedrop = 4;
- const QualityScalerResource* quality_scaler_resource_;
+ const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
const QualityScalerSettings quality_scaler_settings_;
bool has_seen_first_bwe_drop_;
DataRate set_start_bitrate_;
@@ -139,13 +140,16 @@
VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts::
PreventAdaptUpDueToActiveCounts(VideoStreamEncoderResourceManager* manager)
- : manager_(manager) {}
+ : rtc::RefCountedObject<Resource>(), manager_(manager) {}
bool VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts::
IsAdaptationUpAllowed(const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const {
+ rtc::scoped_refptr<Resource> reason_resource) const {
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // ensure that this is running on it instead.
+ RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
VideoAdaptationReason reason =
manager_->GetReasonFromResource(reason_resource);
// We can't adapt up if we're already at the highest setting.
@@ -155,10 +159,11 @@
// TODO(hbos): Why would the reason matter? If a particular resource doesn't
// want us to go up it should prevent us from doing so itself rather than to
// have this catch-all reason- and stats-based approach.
- int num_downgrades = FilterVideoAdaptationCountersByDegradationPreference(
- manager_->active_counts_[reason],
- manager_->effective_degradation_preference_)
- .Total();
+ int num_downgrades =
+ FilterVideoAdaptationCountersByDegradationPreference(
+ manager_->active_counts_[reason],
+ manager_->adaptation_processor_->effective_degradation_preference())
+ .Total();
RTC_DCHECK_GE(num_downgrades, 0);
return num_downgrades > 0;
}
@@ -167,14 +172,17 @@
PreventIncreaseResolutionDueToBitrateResource::
PreventIncreaseResolutionDueToBitrateResource(
VideoStreamEncoderResourceManager* manager)
- : manager_(manager) {}
+ : rtc::RefCountedObject<Resource>(), manager_(manager) {}
bool VideoStreamEncoderResourceManager::
PreventIncreaseResolutionDueToBitrateResource::IsAdaptationUpAllowed(
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const {
+ rtc::scoped_refptr<Resource> reason_resource) const {
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // ensure that this is running on it instead.
+ RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
VideoAdaptationReason reason =
manager_->GetReasonFromResource(reason_resource);
// If increasing resolution due to kQuality, make sure bitrate limits are not
@@ -205,13 +213,16 @@
VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource::
PreventAdaptUpInBalancedResource(VideoStreamEncoderResourceManager* manager)
- : manager_(manager) {}
+ : rtc::RefCountedObject<Resource>(), manager_(manager) {}
bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource::
IsAdaptationUpAllowed(const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const {
+ rtc::scoped_refptr<Resource> reason_resource) const {
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // ensure that this is running on it instead.
+ RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
VideoAdaptationReason reason =
manager_->GetReasonFromResource(reason_resource);
// Don't adapt if BalancedDegradationSettings applies and determines this will
@@ -219,7 +230,7 @@
// TODO(hbos): Why are we allowing violating balanced settings if adapting due
// CPU? Shouldn't this condition be checked regardless of reason?
if (reason == VideoAdaptationReason::kQuality &&
- manager_->effective_degradation_preference_ ==
+ manager_->adaptation_processor_->effective_degradation_preference() ==
DegradationPreference::BALANCED &&
!manager_->balanced_settings_.CanAdaptUp(
input_state.video_codec_type(),
@@ -240,26 +251,29 @@
VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
VideoStreamInputStateProvider* input_state_provider,
- ResourceAdaptationProcessorInterface* adaptation_processor,
VideoStreamEncoderObserver* encoder_stats_observer,
Clock* clock,
bool experiment_cpu_load_estimator,
std::unique_ptr<OveruseFrameDetector> overuse_detector)
- : prevent_adapt_up_due_to_active_counts_(this),
- prevent_increase_resolution_due_to_bitrate_resource_(this),
- prevent_adapt_up_in_balanced_resource_(this),
- encode_usage_resource_(std::move(overuse_detector)),
- quality_scaler_resource_(adaptation_processor),
+ : prevent_adapt_up_due_to_active_counts_(
+ new PreventAdaptUpDueToActiveCounts(this)),
+ prevent_increase_resolution_due_to_bitrate_resource_(
+ new PreventIncreaseResolutionDueToBitrateResource(this)),
+ prevent_adapt_up_in_balanced_resource_(
+ new PreventAdaptUpInBalancedResource(this)),
+ encode_usage_resource_(
+ new EncodeUsageResource(std::move(overuse_detector))),
+ quality_scaler_resource_(new QualityScalerResource()),
+ encoder_queue_(nullptr),
input_state_provider_(input_state_provider),
- adaptation_processor_(adaptation_processor),
+ adaptation_processor_(nullptr),
encoder_stats_observer_(encoder_stats_observer),
degradation_preference_(DegradationPreference::DISABLED),
- effective_degradation_preference_(DegradationPreference::DISABLED),
video_source_restrictions_(),
clock_(clock),
experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
initial_frame_dropper_(
- std::make_unique<InitialFrameDropper>(&quality_scaler_resource_)),
+ std::make_unique<InitialFrameDropper>(quality_scaler_resource_)),
quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
encoder_target_bitrate_bps_(absl::nullopt),
quality_rampup_done_(false),
@@ -267,43 +281,65 @@
encoder_settings_(absl::nullopt),
active_counts_() {
RTC_DCHECK(encoder_stats_observer_);
- MapResourceToReason(&prevent_adapt_up_due_to_active_counts_,
+ MapResourceToReason(prevent_adapt_up_due_to_active_counts_,
VideoAdaptationReason::kQuality);
- MapResourceToReason(&prevent_increase_resolution_due_to_bitrate_resource_,
+ MapResourceToReason(prevent_increase_resolution_due_to_bitrate_resource_,
VideoAdaptationReason::kQuality);
- MapResourceToReason(&prevent_adapt_up_in_balanced_resource_,
+ MapResourceToReason(prevent_adapt_up_in_balanced_resource_,
VideoAdaptationReason::kQuality);
- MapResourceToReason(&encode_usage_resource_, VideoAdaptationReason::kCpu);
- MapResourceToReason(&quality_scaler_resource_,
+ MapResourceToReason(encode_usage_resource_, VideoAdaptationReason::kCpu);
+ MapResourceToReason(quality_scaler_resource_,
VideoAdaptationReason::kQuality);
}
-VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() {
- RTC_DCHECK(!encode_usage_resource_.is_started());
+VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() {}
+
+void VideoStreamEncoderResourceManager::Initialize(
+ rtc::TaskQueue* encoder_queue) {
+ RTC_DCHECK(!encoder_queue_);
+ RTC_DCHECK(encoder_queue);
+ encoder_queue_ = encoder_queue;
+ encode_usage_resource_->Initialize(encoder_queue_);
+ quality_scaler_resource_->Initialize(encoder_queue_);
+}
+
+void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
+ ResourceAdaptationProcessorInterface* adaptation_processor) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ adaptation_processor_ = adaptation_processor;
+ quality_scaler_resource_->SetAdaptationProcessor(adaptation_processor);
}
void VideoStreamEncoderResourceManager::SetDegradationPreferences(
- DegradationPreference degradation_preference,
- DegradationPreference effective_degradation_preference) {
+ DegradationPreference degradation_preference) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
degradation_preference_ = degradation_preference;
- effective_degradation_preference_ = effective_degradation_preference;
UpdateStatsAdaptationSettings();
}
+DegradationPreference
+VideoStreamEncoderResourceManager::degradation_preference() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ return degradation_preference_;
+}
+
void VideoStreamEncoderResourceManager::StartEncodeUsageResource() {
- RTC_DCHECK(!encode_usage_resource_.is_started());
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ RTC_DCHECK(!encode_usage_resource_->is_started());
RTC_DCHECK(encoder_settings_.has_value());
- encode_usage_resource_.StartCheckForOveruse(GetCpuOveruseOptions());
+ encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions());
}
void VideoStreamEncoderResourceManager::StopManagedResources() {
- encode_usage_resource_.StopCheckForOveruse();
- quality_scaler_resource_.StopCheckForOveruse();
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ encode_usage_resource_->StopCheckForOveruse();
+ quality_scaler_resource_->StopCheckForOveruse();
}
void VideoStreamEncoderResourceManager::MapResourceToReason(
- Resource* resource,
+ rtc::scoped_refptr<Resource> resource,
VideoAdaptationReason reason) {
+ rtc::CritScope crit(&resource_lock_);
RTC_DCHECK(resource);
RTC_DCHECK(absl::c_find_if(resources_,
[resource](const ResourceAndReason& r) {
@@ -313,22 +349,25 @@
resources_.emplace_back(resource, reason);
}
-std::vector<Resource*> VideoStreamEncoderResourceManager::MappedResources()
- const {
- std::vector<Resource*> resources;
+std::vector<rtc::scoped_refptr<Resource>>
+VideoStreamEncoderResourceManager::MappedResources() const {
+ rtc::CritScope crit(&resource_lock_);
+ std::vector<rtc::scoped_refptr<Resource>> resources;
for (auto const& resource_and_reason : resources_) {
resources.push_back(resource_and_reason.resource);
}
return resources;
}
-QualityScalerResource*
+rtc::scoped_refptr<QualityScalerResource>
VideoStreamEncoderResourceManager::quality_scaler_resource_for_testing() {
- return &quality_scaler_resource_;
+ rtc::CritScope crit(&resource_lock_);
+ return quality_scaler_resource_;
}
void VideoStreamEncoderResourceManager::SetEncoderSettings(
EncoderSettings encoder_settings) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_settings_ = std::move(encoder_settings);
quality_rampup_experiment_.SetMaxBitrate(
@@ -339,6 +378,7 @@
void VideoStreamEncoderResourceManager::SetStartBitrate(
DataRate start_bitrate) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
if (!start_bitrate.IsZero())
encoder_target_bitrate_bps_ = start_bitrate.bps();
initial_frame_dropper_->SetStartBitrate(start_bitrate,
@@ -347,6 +387,7 @@
void VideoStreamEncoderResourceManager::SetTargetBitrate(
DataRate target_bitrate) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
if (!target_bitrate.IsZero())
encoder_target_bitrate_bps_ = target_bitrate.bps();
initial_frame_dropper_->SetTargetBitrate(target_bitrate,
@@ -355,10 +396,14 @@
void VideoStreamEncoderResourceManager::SetEncoderRates(
const VideoEncoder::RateControlParameters& encoder_rates) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_rates_ = encoder_rates;
}
void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // PostTask the request to adapt due to frame drop.
adaptation_processor_->TriggerAdaptationDueToFrameDroppedDueToSize(
quality_scaler_resource_);
initial_frame_dropper_->OnFrameDroppedDueToSize();
@@ -367,51 +412,57 @@
void VideoStreamEncoderResourceManager::OnEncodeStarted(
const VideoFrame& cropped_frame,
int64_t time_when_first_seen_us) {
- encode_usage_resource_.OnEncodeStarted(cropped_frame,
- time_when_first_seen_us);
+ encode_usage_resource_->OnEncodeStarted(cropped_frame,
+ time_when_first_seen_us);
}
void VideoStreamEncoderResourceManager::OnEncodeCompleted(
const EncodedImage& encoded_image,
int64_t time_sent_in_us,
absl::optional<int> encode_duration_us) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
// Inform |encode_usage_resource_| of the encode completed event.
uint32_t timestamp = encoded_image.Timestamp();
int64_t capture_time_us =
encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
- encode_usage_resource_.OnEncodeCompleted(timestamp, time_sent_in_us,
- capture_time_us, encode_duration_us);
+ encode_usage_resource_->OnEncodeCompleted(
+ timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
// Inform |quality_scaler_resource_| of the encode completed event.
- quality_scaler_resource_.OnEncodeCompleted(encoded_image, time_sent_in_us);
+ quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
}
void VideoStreamEncoderResourceManager::OnFrameDropped(
EncodedImageCallback::DropReason reason) {
- quality_scaler_resource_.OnFrameDropped(reason);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ quality_scaler_resource_->OnFrameDropped(reason);
}
bool VideoStreamEncoderResourceManager::DropInitialFrames() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
return initial_frame_dropper_->DropInitialFrames();
}
void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
initial_frame_dropper_->OnMaybeEncodeFrame();
MaybePerformQualityRampupExperiment();
}
void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
if (qp_thresholds.has_value()) {
- quality_scaler_resource_.StopCheckForOveruse();
- quality_scaler_resource_.StartCheckForOveruse(qp_thresholds.value());
+ quality_scaler_resource_->StopCheckForOveruse();
+ quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value());
} else {
- quality_scaler_resource_.StopCheckForOveruse();
+ quality_scaler_resource_->StopCheckForOveruse();
}
initial_frame_dropper_->OnQualityScalerSettingsUpdated();
}
void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
const auto scaling_settings = encoder_info.scaling_settings;
const bool quality_scaling_allowed =
IsResolutionScalingEnabled(degradation_preference_) &&
@@ -420,7 +471,7 @@
// TODO(https://crbug.com/webrtc/11222): Should this move to
// QualityScalerResource?
if (quality_scaling_allowed) {
- if (!quality_scaler_resource_.is_started()) {
+ if (!quality_scaler_resource_->is_started()) {
// Quality scaler has not already been configured.
// Use experimental thresholds if available.
@@ -439,26 +490,27 @@
// Set the qp-thresholds to the balanced settings if balanced mode.
if (degradation_preference_ == DegradationPreference::BALANCED &&
- quality_scaler_resource_.is_started()) {
+ quality_scaler_resource_->is_started()) {
absl::optional<VideoEncoder::QpThresholds> thresholds =
balanced_settings_.GetQpThresholds(
GetVideoCodecTypeOrGeneric(encoder_settings_),
LastInputFrameSizeOrDefault());
if (thresholds) {
- quality_scaler_resource_.SetQpThresholds(*thresholds);
+ quality_scaler_resource_->SetQpThresholds(*thresholds);
}
}
UpdateStatsAdaptationSettings();
}
VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource(
- const Resource& resource) const {
+ rtc::scoped_refptr<Resource> resource) const {
+ rtc::CritScope crit(&resource_lock_);
const auto& registered_resource =
absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) {
- return r.resource == &resource;
+ return r.resource == resource;
});
RTC_DCHECK(registered_resource != resources_.end())
- << resource.name() << " not found.";
+ << resource->name() << " not found.";
return registered_resource->reason;
}
@@ -468,6 +520,7 @@
// remotely cope with the load right now.
CpuOveruseOptions VideoStreamEncoderResourceManager::GetCpuOveruseOptions()
const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
// This is already ensured by the only caller of this method:
// StartResourceAdaptation().
RTC_DCHECK(encoder_settings_.has_value());
@@ -485,6 +538,7 @@
}
int VideoStreamEncoderResourceManager::LastInputFrameSizeOrDefault() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
return input_state_provider_->InputState().frame_size_pixels().value_or(
kDefaultInputPixelsWidth * kDefaultInputPixelsHeight);
}
@@ -492,7 +546,11 @@
void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
- const Resource* reason) {
+ rtc::scoped_refptr<Resource> reason) {
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // ensure that this is running on it instead, and PostTask back to the encoder
+ // queue if need be.
+ RTC_DCHECK_RUN_ON(encoder_queue_);
video_source_restrictions_ = restrictions;
VideoAdaptationCounters previous_adaptation_counters =
active_counts_[VideoAdaptationReason::kQuality] +
@@ -504,7 +562,7 @@
// to be updated every time the adaptation counter is incremented or
// decremented due to a resource.
RTC_DCHECK_EQ(adaptation_counters_total_abs_diff, 1);
- VideoAdaptationReason reason_type = GetReasonFromResource(*reason);
+ VideoAdaptationReason reason_type = GetReasonFromResource(reason);
UpdateAdaptationStats(adaptation_counters, reason_type);
} else if (adaptation_counters.Total() == 0) {
// Adaptation was manually reset - clear the per-reason counters too.
@@ -521,6 +579,7 @@
}
void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
absl::optional<double> codec_max_frame_rate =
encoder_settings_.has_value()
? absl::optional<double>(
@@ -537,7 +596,7 @@
codec_max_frame_rate.value() < target_frame_rate.value())) {
target_frame_rate = codec_max_frame_rate;
}
- encode_usage_resource_.SetTargetFrameRate(target_frame_rate);
+ encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
}
void VideoStreamEncoderResourceManager::OnAdaptationCountChanged(
@@ -606,6 +665,7 @@
void VideoStreamEncoderResourceManager::UpdateAdaptationStats(
const VideoAdaptationCounters& total_counts,
VideoAdaptationReason reason) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
// Update active counts
VideoAdaptationCounters& active_count = active_counts_[reason];
VideoAdaptationCounters& other_active = active_counts_[OtherReason(reason)];
@@ -618,12 +678,13 @@
}
void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
VideoStreamEncoderObserver::AdaptationSettings cpu_settings(
IsResolutionScalingEnabled(degradation_preference_),
IsFramerateScalingEnabled(degradation_preference_));
VideoStreamEncoderObserver::AdaptationSettings quality_settings =
- quality_scaler_resource_.is_started()
+ quality_scaler_resource_->is_started()
? cpu_settings
: VideoStreamEncoderObserver::AdaptationSettings();
encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings,
@@ -631,7 +692,8 @@
}
void VideoStreamEncoderResourceManager::MaybePerformQualityRampupExperiment() {
- if (!quality_scaler_resource_.is_started())
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ if (!quality_scaler_resource_->is_started())
return;
if (quality_rampup_done_)
@@ -648,7 +710,7 @@
if (encoder_settings_ &&
encoder_target_bitrate_bps_.value_or(0) ==
encoder_settings_->video_codec().maxBitrate * 1000 &&
- quality_scaler_resource_.QpFastFilterLow()) {
+ quality_scaler_resource_->QpFastFilterLow()) {
try_quality_rampup = true;
}
}
@@ -667,12 +729,14 @@
}
void VideoStreamEncoderResourceManager::ResetActiveCounts() {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
active_counts_.clear();
active_counts_[VideoAdaptationReason::kCpu] = VideoAdaptationCounters();
active_counts_[VideoAdaptationReason::kQuality] = VideoAdaptationCounters();
}
std::string VideoStreamEncoderResourceManager::ActiveCountsToString() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK_EQ(2, active_counts_.size());
rtc::StringBuilder ss;
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h
index d0e5455..43364a8 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.h
+++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -20,6 +20,7 @@
#include "absl/types/optional.h"
#include "api/rtp_parameters.h"
+#include "api/scoped_refptr.h"
#include "api/video/video_adaptation_counters.h"
#include "api/video/video_adaptation_reason.h"
#include "api/video/video_frame.h"
@@ -35,6 +36,7 @@
#include "rtc_base/experiments/quality_rampup_experiment.h"
#include "rtc_base/experiments/quality_scaler_settings.h"
#include "rtc_base/strings/string_builder.h"
+#include "rtc_base/task_queue.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/encode_usage_resource.h"
#include "video/adaptation/overuse_frame_detector.h"
@@ -61,16 +63,23 @@
public:
VideoStreamEncoderResourceManager(
VideoStreamInputStateProvider* input_state_provider,
- ResourceAdaptationProcessorInterface* adaptation_processor,
VideoStreamEncoderObserver* encoder_stats_observer,
Clock* clock,
bool experiment_cpu_load_estimator,
std::unique_ptr<OveruseFrameDetector> overuse_detector);
~VideoStreamEncoderResourceManager() override;
- void SetDegradationPreferences(
- DegradationPreference degradation_preference,
- DegradationPreference effective_degradation_preference);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // pass it in here.
+ void Initialize(rtc::TaskQueue* encoder_queue);
+ void SetAdaptationProcessor(
+ ResourceAdaptationProcessorInterface* adaptation_processor);
+
+ // TODO(https://crbug.com/webrtc/11563): The degradation preference is a
+ // setting of the Processor, it does not belong to the Manager - can we get
+ // rid of this?
+ void SetDegradationPreferences(DegradationPreference degradation_preference);
+ DegradationPreference degradation_preference() const;
// Starts the encode usage resource. The quality scaler resource is
// automatically started on being configured.
@@ -103,9 +112,11 @@
// - Legacy getStats() purposes.
// - Preventing adapting up in some circumstances (which may be questionable).
// TODO(hbos): Can we get rid of this?
- void MapResourceToReason(Resource* resource, VideoAdaptationReason reason);
- std::vector<Resource*> MappedResources() const;
- QualityScalerResource* quality_scaler_resource_for_testing();
+ void MapResourceToReason(rtc::scoped_refptr<Resource> resource,
+ VideoAdaptationReason reason);
+ std::vector<rtc::scoped_refptr<Resource>> MappedResources() const;
+ rtc::scoped_refptr<QualityScalerResource>
+ quality_scaler_resource_for_testing();
// If true, the VideoStreamEncoder should eexecute its logic to maybe drop
// frames baseed on size and bitrate.
bool DropInitialFrames() const;
@@ -115,7 +126,7 @@
void OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
- const Resource* reason) override;
+ rtc::scoped_refptr<Resource> reason) override;
// For reasons of adaptation and statistics, we not only count the total
// number of adaptations, but we also count the number of adaptations per
@@ -132,15 +143,12 @@
private:
class InitialFrameDropper;
- VideoAdaptationReason GetReasonFromResource(const Resource& resource) const;
+ VideoAdaptationReason GetReasonFromResource(
+ rtc::scoped_refptr<Resource> resource) const;
CpuOveruseOptions GetCpuOveruseOptions() const;
int LastInputFrameSizeOrDefault() const;
- // Makes |video_source_restrictions_| up-to-date and informs the
- // |adaptation_listener_| if restrictions are changed, allowing the listener
- // to reconfigure the source accordingly.
- void MaybeUpdateVideoSourceRestrictions(const Resource* reason_resource);
// Calculates an up-to-date value of the target frame rate and informs the
// |encode_usage_resource_| of the new value.
void MaybeUpdateTargetFrameRate();
@@ -168,7 +176,8 @@
// Does not trigger adaptations, only prevents adapting up based on
// |active_counts_|.
- class PreventAdaptUpDueToActiveCounts final : public Resource {
+ class PreventAdaptUpDueToActiveCounts final
+ : public rtc::RefCountedObject<Resource> {
public:
explicit PreventAdaptUpDueToActiveCounts(
VideoStreamEncoderResourceManager* manager);
@@ -182,14 +191,15 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const override;
+ rtc::scoped_refptr<Resource> reason_resource) const override;
private:
VideoStreamEncoderResourceManager* manager_;
- } prevent_adapt_up_due_to_active_counts_;
+ };
// Does not trigger adaptations, only prevents adapting up resolution.
- class PreventIncreaseResolutionDueToBitrateResource final : public Resource {
+ class PreventIncreaseResolutionDueToBitrateResource final
+ : public rtc::RefCountedObject<Resource> {
public:
explicit PreventIncreaseResolutionDueToBitrateResource(
VideoStreamEncoderResourceManager* manager);
@@ -203,14 +213,15 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const override;
+ rtc::scoped_refptr<Resource> reason_resource) const override;
private:
VideoStreamEncoderResourceManager* manager_;
- } prevent_increase_resolution_due_to_bitrate_resource_;
+ };
// Does not trigger adaptations, only prevents adapting up in BALANCED.
- class PreventAdaptUpInBalancedResource final : public Resource {
+ class PreventAdaptUpInBalancedResource final
+ : public rtc::RefCountedObject<Resource> {
public:
explicit PreventAdaptUpInBalancedResource(
VideoStreamEncoderResourceManager* manager);
@@ -224,54 +235,74 @@
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
- const Resource& reason_resource) const override;
+ rtc::scoped_refptr<Resource> reason_resource) const override;
private:
VideoStreamEncoderResourceManager* manager_;
- } prevent_adapt_up_in_balanced_resource_;
+ };
- EncodeUsageResource encode_usage_resource_;
- QualityScalerResource quality_scaler_resource_;
+ const rtc::scoped_refptr<PreventAdaptUpDueToActiveCounts>
+ prevent_adapt_up_due_to_active_counts_;
+ const rtc::scoped_refptr<PreventIncreaseResolutionDueToBitrateResource>
+ prevent_increase_resolution_due_to_bitrate_resource_;
+ const rtc::scoped_refptr<PreventAdaptUpInBalancedResource>
+ prevent_adapt_up_in_balanced_resource_;
+ const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;
+ const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
- VideoStreamInputStateProvider* const input_state_provider_;
- ResourceAdaptationProcessorInterface* const adaptation_processor_;
- VideoStreamEncoderObserver* const encoder_stats_observer_;
+ rtc::TaskQueue* encoder_queue_;
+ VideoStreamInputStateProvider* const input_state_provider_
+ RTC_GUARDED_BY(encoder_queue_);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // guard the processor by it instead.
+ ResourceAdaptationProcessorInterface* adaptation_processor_
+ RTC_GUARDED_BY(encoder_queue_);
+ VideoStreamEncoderObserver* const encoder_stats_observer_
+ RTC_GUARDED_BY(encoder_queue_);
- DegradationPreference degradation_preference_;
- DegradationPreference effective_degradation_preference_;
- VideoSourceRestrictions video_source_restrictions_;
+ DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_);
+ VideoSourceRestrictions video_source_restrictions_
+ RTC_GUARDED_BY(encoder_queue_);
- const BalancedDegradationSettings balanced_settings_;
- Clock* clock_;
- const bool experiment_cpu_load_estimator_;
- const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_;
- const bool quality_scaling_experiment_enabled_;
- absl::optional<uint32_t> encoder_target_bitrate_bps_;
- absl::optional<VideoEncoder::RateControlParameters> encoder_rates_;
- bool quality_rampup_done_;
- QualityRampupExperiment quality_rampup_experiment_;
- absl::optional<EncoderSettings> encoder_settings_;
+ const BalancedDegradationSettings balanced_settings_
+ RTC_GUARDED_BY(encoder_queue_);
+ Clock* clock_ RTC_GUARDED_BY(encoder_queue_);
+ const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_);
+ const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_
+ RTC_GUARDED_BY(encoder_queue_);
+ const bool quality_scaling_experiment_enabled_ RTC_GUARDED_BY(encoder_queue_);
+ absl::optional<uint32_t> encoder_target_bitrate_bps_
+ RTC_GUARDED_BY(encoder_queue_);
+ absl::optional<VideoEncoder::RateControlParameters> encoder_rates_
+ RTC_GUARDED_BY(encoder_queue_);
+ bool quality_rampup_done_ RTC_GUARDED_BY(encoder_queue_);
+ QualityRampupExperiment quality_rampup_experiment_
+ RTC_GUARDED_BY(encoder_queue_);
+ absl::optional<EncoderSettings> encoder_settings_
+ RTC_GUARDED_BY(encoder_queue_);
// Ties a resource to a reason for statistical reporting. This AdaptReason is
// also used by this module to make decisions about how to adapt up/down.
struct ResourceAndReason {
- ResourceAndReason(Resource* resource, VideoAdaptationReason reason)
+ ResourceAndReason(rtc::scoped_refptr<Resource> resource,
+ VideoAdaptationReason reason)
: resource(resource), reason(reason) {}
virtual ~ResourceAndReason() = default;
- Resource* const resource;
+ const rtc::scoped_refptr<Resource> resource;
const VideoAdaptationReason reason;
};
- std::vector<ResourceAndReason> resources_;
+ rtc::CriticalSection resource_lock_;
+ std::vector<ResourceAndReason> resources_ RTC_GUARDED_BY(&resource_lock_);
// One AdaptationCounter for each reason, tracking the number of times we have
// adapted for each reason. The sum of active_counts_ MUST always equal the
// total adaptation provided by the VideoSourceRestrictions.
- // TODO(https://crbug.com/webrtc/11392): Move all active count logic to
- // encoder_stats_observer_; Counters used for deciding if the video resolution
- // or framerate is currently restricted, and if so, why, on a per degradation
- // preference basis.
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // guard the activec counts by it instead. The |encoder_stats_observer_| is
+ // thread-safe anyway, and active counts are used by
+ // PreventAdaptUpDueToActiveCounts to make decisions.
std::unordered_map<VideoAdaptationReason, VideoAdaptationCounters>
- active_counts_;
+ active_counts_ RTC_GUARDED_BY(encoder_queue_);
};
} // namespace webrtc
diff --git a/video/video_source_sink_controller.h b/video/video_source_sink_controller.h
index 4811b286..68fef3f 100644
--- a/video/video_source_sink_controller.h
+++ b/video/video_source_sink_controller.h
@@ -53,9 +53,6 @@
rtc::VideoSinkWants CurrentSettingsToSinkWants() const
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
- // TODO(hbos): If everything is handled on the same sequence (i.e.
- // VideoStreamEncoder's encoder queue) then |crit_| can be replaced by
- // sequence checker. Investigate if we want to do this.
mutable rtc::CriticalSection crit_;
rtc::VideoSinkInterface<VideoFrame>* const sink_;
rtc::VideoSourceInterface<VideoFrame>* source_ RTC_GUARDED_BY(&crit_);
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index c69cf1e..bb779ff 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -262,7 +262,6 @@
&input_state_provider_,
encoder_stats_observer)),
stream_resource_manager_(&input_state_provider_,
- resource_adaptation_processor_.get(),
encoder_stats_observer,
clock_,
settings_.experiment_cpu_load_estimator,
@@ -275,13 +274,25 @@
RTC_DCHECK(encoder_stats_observer);
RTC_DCHECK_GE(number_of_cores, 1);
- resource_adaptation_processor_->AddAdaptationListener(
- &stream_resource_manager_);
- resource_adaptation_processor_->AddAdaptationListener(this);
+ stream_resource_manager_.Initialize(&encoder_queue_);
- // Add the stream resource manager's resources to the processor.
- for (Resource* resource : stream_resource_manager_.MappedResources())
- resource_adaptation_processor_->AddResource(resource);
+ rtc::Event initialize_processor_event;
+ encoder_queue_.PostTask([this, &initialize_processor_event] {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // initialize the processor on it instead.
+ resource_adaptation_processor_->InitializeOnResourceAdaptationQueue();
+ stream_resource_manager_.SetAdaptationProcessor(
+ resource_adaptation_processor_.get());
+ resource_adaptation_processor_->AddAdaptationListener(
+ &stream_resource_manager_);
+ resource_adaptation_processor_->AddAdaptationListener(this);
+ // Add the stream resource manager's resources to the processor.
+ for (Resource* resource : stream_resource_manager_.MappedResources())
+ resource_adaptation_processor_->AddResource(resource);
+ initialize_processor_event.Set();
+ });
+ initialize_processor_event.Wait(rtc::Event::kForever);
for (auto& state : encoder_buffer_state_)
state.fill(std::numeric_limits<int64_t>::max());
@@ -298,14 +309,25 @@
video_source_sink_controller_.SetSource(nullptr);
encoder_queue_.PostTask([this] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // destroy the processor on it instead.
+ if (resource_adaptation_processor_) {
+ resource_adaptation_processor_->StopResourceAdaptation();
+ for (Resource* resource : stream_resource_manager_.MappedResources()) {
+ resource_adaptation_processor_->RemoveResource(resource);
+ }
+ resource_adaptation_processor_->RemoveAdaptationListener(this);
+ resource_adaptation_processor_->RemoveAdaptationListener(
+ &stream_resource_manager_);
+ stream_resource_manager_.SetAdaptationProcessor(nullptr);
+ resource_adaptation_processor_.reset();
+ }
stream_resource_manager_.StopManagedResources();
- resource_adaptation_processor_->StopResourceAdaptation();
rate_allocator_ = nullptr;
bitrate_observer_ = nullptr;
ReleaseEncoder();
shutdown_event_.Set();
});
-
shutdown_event_.Wait(rtc::Event::kForever);
}
@@ -336,14 +358,18 @@
const DegradationPreference& degradation_preference) {
RTC_DCHECK_RUN_ON(&thread_checker_);
video_source_sink_controller_.SetSource(source);
- encoder_queue_.PostTask([this, source, degradation_preference] {
+ input_state_provider_.OnHasInputChanged(source);
+ encoder_queue_.PostTask([this, degradation_preference] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- input_state_provider_.OnHasInputChanged(source);
+ if (!resource_adaptation_processor_) {
+ // The VideoStreamEncoder was stopped and the processor destroyed before
+ // this task had a chance to execute. No action needed.
+ return;
+ }
resource_adaptation_processor_->SetDegradationPreference(
degradation_preference);
stream_resource_manager_.SetDegradationPreferences(
- resource_adaptation_processor_->degradation_preference(),
- resource_adaptation_processor_->effective_degradation_preference());
+ resource_adaptation_processor_->degradation_preference());
if (encoder_) {
stream_resource_manager_.ConfigureQualityScaler(
encoder_->GetEncoderInfo());
@@ -670,10 +696,18 @@
}
if (pending_encoder_creation_) {
+ // TODO(hbos): Stopping and restarting for backwards compatibility reasons.
+ // We may be able to change this to "EnsureStarted()" if it took care of
+ // reconfiguring the QualityScaler as well. (ConfigureQualityScaler() is
+ // invoked later in this method.)
stream_resource_manager_.StopManagedResources();
- resource_adaptation_processor_->StopResourceAdaptation();
stream_resource_manager_.StartEncodeUsageResource();
- resource_adaptation_processor_->StartResourceAdaptation();
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // PostTask ensuring it is started.
+ if (resource_adaptation_processor_) {
+ // Ensures started. If already started this is a NO-OP.
+ resource_adaptation_processor_->StartResourceAdaptation();
+ }
pending_encoder_creation_ = false;
}
@@ -755,8 +789,7 @@
resource_adaptation_processor_->SetIsScreenshare(
encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen);
stream_resource_manager_.SetDegradationPreferences(
- resource_adaptation_processor_->degradation_preference(),
- resource_adaptation_processor_->effective_degradation_preference());
+ resource_adaptation_processor_->degradation_preference());
input_state_provider_.OnEncoderSettingsChanged(encoder_settings);
stream_resource_manager_.SetEncoderSettings(encoder_settings);
}
@@ -1683,7 +1716,9 @@
void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
- const Resource* reason) {
+ rtc::scoped_refptr<Resource> reason) {
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // ensure that this is running on it instead.
RTC_DCHECK_RUN_ON(&encoder_queue_);
video_source_sink_controller_.SetRestrictions(std::move(restrictions));
video_source_sink_controller_.PushSourceSinkSettings();
@@ -1887,7 +1922,7 @@
if (!automatic_animation_detection_experiment_.enabled ||
encoder_config_.content_type !=
VideoEncoderConfig::ContentType::kScreen ||
- resource_adaptation_processor_->degradation_preference() !=
+ stream_resource_manager_.degradation_preference() !=
DegradationPreference::BALANCED) {
return;
}
@@ -1952,13 +1987,19 @@
}
}
void VideoStreamEncoder::InjectAdaptationResource(
- Resource* resource,
+ rtc::scoped_refptr<Resource> resource,
VideoAdaptationReason reason) {
- stream_resource_manager_.MapResourceToReason(resource, reason);
- resource_adaptation_processor_->AddResource(resource);
+ rtc::Event inject_resource_event;
+ encoder_queue_.PostTask([this, resource, reason, &inject_resource_event] {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
+ stream_resource_manager_.MapResourceToReason(resource, reason);
+ resource_adaptation_processor_->AddResource(resource);
+ inject_resource_event.Set();
+ });
+ inject_resource_event.Wait(rtc::Event::kForever);
}
-QualityScalerResource*
+rtc::scoped_refptr<QualityScalerResource>
VideoStreamEncoder::quality_scaler_resource_for_testing() {
RTC_DCHECK_RUN_ON(&encoder_queue_);
return stream_resource_manager_.quality_scaler_resource_for_testing();
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index a4a055d..6194355 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -110,15 +110,16 @@
void OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
- const Resource* reason) override;
+ rtc::scoped_refptr<Resource> reason) override;
// Used for injected test resources.
// TODO(eshr): Move all adaptation tests out of VideoStreamEncoder tests.
- void InjectAdaptationResource(Resource* resource,
+ void InjectAdaptationResource(rtc::scoped_refptr<Resource> resource,
VideoAdaptationReason reason)
RTC_RUN_ON(&encoder_queue_);
- QualityScalerResource* quality_scaler_resource_for_testing();
+ rtc::scoped_refptr<QualityScalerResource>
+ quality_scaler_resource_for_testing();
private:
class VideoFrameInfo {
@@ -400,27 +401,30 @@
bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_);
// Provies video stream input states: current resolution and frame rate.
- VideoStreamInputStateProvider input_state_provider_
- RTC_GUARDED_BY(&encoder_queue_);
+ // This class is thread-safe.
+ VideoStreamInputStateProvider input_state_provider_;
// Responsible for adapting input resolution or frame rate to ensure resources
// (e.g. CPU or bandwidth) are not overused.
+ // This class is single-threaded.
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // guard the processor by it instead.
std::unique_ptr<ResourceAdaptationProcessorInterface>
resource_adaptation_processor_ RTC_GUARDED_BY(&encoder_queue_);
// Handles input, output and stats reporting related to VideoStreamEncoder
// specific resources, such as "encode usage percent" measurements and "QP
// scaling". Also involved with various mitigations such as inital frame
// dropping.
+ // The manager primarily operates on the |encoder_queue_| but its lifetime is
+ // tied to the VideoStreamEncoder (which is destroyed off the encoder queue)
+ // and its resource list is accessible from any thread.
+ // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
+ // remove the RTC_GUARDED_BY to get resources on the adaptation queue.
VideoStreamEncoderResourceManager stream_resource_manager_
RTC_GUARDED_BY(&encoder_queue_);
// Carries out the VideoSourceRestrictions provided by the
// ResourceAdaptationProcessor, i.e. reconfigures the source of video frames
// to provide us with different resolution or frame rate.
- //
- // Used on the |encoder_queue_| with a few exceptions:
- // - VideoStreamEncoder::SetSource() invokes SetSource().
- // - VideoStreamEncoder::SetSink() invokes SetRotationApplied() and
- // PushSourceSinkSettings().
- // - VideoStreamEncoder::Stop() invokes SetSource().
+ // This class is thread-safe.
VideoSourceSinkController video_source_sink_controller_;
// All public methods are proxied to |encoder_queue_|. It must must be
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index cf9e790..30660bb 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -178,13 +178,11 @@
overuse_detector_proxy_ =
new CpuOveruseDetectorProxy(stats_proxy)),
task_queue_factory),
- fake_cpu_resource_(std::make_unique<FakeResource>("FakeResource[CPU]")),
- fake_quality_resource_(
- std::make_unique<FakeResource>("FakeResource[QP]")) {
- InjectAdaptationResource(fake_quality_resource_.get(),
+ fake_cpu_resource_(new FakeResource("FakeResource[CPU]")),
+ fake_quality_resource_(new FakeResource("FakeResource[QP]")) {
+ InjectAdaptationResource(fake_quality_resource_,
VideoAdaptationReason::kQuality);
- InjectAdaptationResource(fake_cpu_resource_.get(),
- VideoAdaptationReason::kCpu);
+ InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
}
// This is used as a synchronisation mechanism, to make sure that the
@@ -248,8 +246,8 @@
}
CpuOveruseDetectorProxy* overuse_detector_proxy_;
- std::unique_ptr<FakeResource> fake_cpu_resource_;
- std::unique_ptr<FakeResource> fake_quality_resource_;
+ rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
+ rtc::scoped_refptr<FakeResource> fake_quality_resource_;
};
class VideoStreamFactory
