[Adaptation] Introducing call/adaptation/ResourceAdaptationProcessor. This CL is part of the Call-Level Adaptation Processing design doc: https://docs.google.com/document/d/1ZyC26yOCknrrcYa839ZWLxD6o6Gig5A3lVTh4E41074/edit?usp=sharing This gets to the heart of unblocking call-level adaptation, largely made possible due to the previous CLs in the chain. The parts of the code that are responsible for responding to resource usage signals, obtaining adaptations and applying them are moved to ResourceAdaptationProcessor in call/adaptation/. The parts of the code that are responsible for managing VideoStreamEncoder-specific resources stay inside the VideoStreamEncoderResourceManager class in video/adaptation/. After this CL lands it should soon be possible to move the Processor over to a separate task queue and let the Manager stay on the encoder queue if PostTasks are added for communication between the two objects. Bug: webrtc:11172 Change-Id: Ifa212467b4afd16e7ebfb9adfe17d2dca1cb7d67 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/173021 Commit-Queue: Henrik Boström <hbos@webrtc.org> Reviewed-by: Evan Shrubsole <eshr@google.com> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Cr-Commit-Position: refs/heads/master@{#31105}
diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index 24dee6a..94656cf 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn
@@ -14,6 +14,8 @@ "encoder_settings.h", "resource.cc", "resource.h", + "resource_adaptation_processor.cc", + "resource_adaptation_processor.h", "resource_adaptation_processor_interface.cc", "resource_adaptation_processor_interface.h", "video_source_restrictions.cc",
diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/resource_adaptation_processor.cc new file mode 100644 index 0000000..d75f62b --- /dev/null +++ b/call/adaptation/resource_adaptation_processor.cc
@@ -0,0 +1,209 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/resource_adaptation_processor.h" + +#include <utility> + +#include "absl/algorithm/container.h" + +namespace webrtc { + +ResourceAdaptationProcessor::ResourceAdaptationProcessor( + VideoStreamInputStateProvider* input_state_provider, + VideoStreamEncoderObserver* encoder_stats_observer) + : input_state_provider_(input_state_provider), + encoder_stats_observer_(encoder_stats_observer), + resources_(), + degradation_preference_(DegradationPreference::DISABLED), + effective_degradation_preference_(DegradationPreference::DISABLED), + is_screenshare_(false), + stream_adapter_(std::make_unique<VideoStreamAdapter>()), + last_reported_source_restrictions_() {} + +ResourceAdaptationProcessor::~ResourceAdaptationProcessor() {} + +DegradationPreference ResourceAdaptationProcessor::degradation_preference() + const { + return degradation_preference_; +} + +DegradationPreference +ResourceAdaptationProcessor::effective_degradation_preference() const { + return effective_degradation_preference_; +} + +void ResourceAdaptationProcessor::StartResourceAdaptation() { + for (auto* resource : resources_) { + resource->RegisterListener(this); + } +} + +void ResourceAdaptationProcessor::StopResourceAdaptation() { + for (auto* resource : resources_) { + resource->UnregisterListener(this); + } +} + +void ResourceAdaptationProcessor::AddAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) { + adaptation_listeners_.push_back(adaptation_listener); +} + +void ResourceAdaptationProcessor::AddResource(Resource* resource) { + resources_.push_back(resource); +} + +void ResourceAdaptationProcessor::SetDegradationPreference( + DegradationPreference degradation_preference) { + degradation_preference_ = degradation_preference; + MaybeUpdateEffectiveDegradationPreference(); +} + +void ResourceAdaptationProcessor::SetIsScreenshare(bool is_screenshare) { + is_screenshare_ = is_screenshare; + MaybeUpdateEffectiveDegradationPreference(); +} + +void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { + effective_degradation_preference_ = + (is_screenshare_ && + degradation_preference_ == DegradationPreference::BALANCED) + ? DegradationPreference::MAINTAIN_RESOLUTION + : degradation_preference_; + stream_adapter_->SetDegradationPreference(effective_degradation_preference_); + MaybeUpdateVideoSourceRestrictions(nullptr); +} + +void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { + stream_adapter_->ClearRestrictions(); + MaybeUpdateVideoSourceRestrictions(nullptr); +} + +void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions( + const Resource* reason) { + VideoSourceRestrictions new_soure_restrictions = + FilterRestrictionsByDegradationPreference( + stream_adapter_->source_restrictions(), + effective_degradation_preference_); + if (last_reported_source_restrictions_ != new_soure_restrictions) { + last_reported_source_restrictions_ = std::move(new_soure_restrictions); + for (auto* adaptation_listener : adaptation_listeners_) { + adaptation_listener->OnVideoSourceRestrictionsUpdated( + last_reported_source_restrictions_, + stream_adapter_->adaptation_counters(), reason); + } + } +} + +ResourceListenerResponse +ResourceAdaptationProcessor::OnResourceUsageStateMeasured( + const Resource& resource) { + switch (resource.usage_state()) { + case ResourceUsageState::kOveruse: + return OnResourceOveruse(resource); + case ResourceUsageState::kStable: + // TODO(https://crbug.com/webrtc/11172): Delete kStable in favor of null. + RTC_NOTREACHED(); + return ResourceListenerResponse::kNothing; + case ResourceUsageState::kUnderuse: + OnResourceUnderuse(resource); + return ResourceListenerResponse::kNothing; + } +} + +bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation( + const VideoStreamInputState& input_state) const { + return input_state.HasInputFrameSizeAndFramesPerSecond() && + (effective_degradation_preference_ != + DegradationPreference::MAINTAIN_RESOLUTION || + input_state.frames_per_second() >= kMinFrameRateFps); +} + +void ResourceAdaptationProcessor::OnResourceUnderuse( + const Resource& reason_resource) { + VideoStreamInputState input_state = input_state_provider_->InputState(); + if (effective_degradation_preference_ == DegradationPreference::DISABLED || + !HasSufficientInputForAdaptation(input_state)) { + return; + } + // Update video input states and encoder settings for accurate adaptation. + stream_adapter_->SetInput(input_state); + // How can this stream be adapted up? + Adaptation adaptation = stream_adapter_->GetAdaptationUp(); + if (adaptation.status() != Adaptation::Status::kValid) + return; + // Are all resources OK with this adaptation being applied? + VideoSourceRestrictions restrictions_before = + stream_adapter_->source_restrictions(); + VideoSourceRestrictions restrictions_after = + stream_adapter_->PeekNextRestrictions(adaptation); + if (!absl::c_all_of(resources_, [&input_state, &restrictions_before, + &restrictions_after, + &reason_resource](const Resource* resource) { + return resource->IsAdaptationUpAllowed(input_state, restrictions_before, + restrictions_after, + reason_resource); + })) { + return; + } + // Apply adaptation. + stream_adapter_->ApplyAdaptation(adaptation); + // Update VideoSourceRestrictions based on adaptation. This also informs the + // |adaptation_listeners_|. + MaybeUpdateVideoSourceRestrictions(&reason_resource); +} + +ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse( + const Resource& reason_resource) { + VideoStreamInputState input_state = input_state_provider_->InputState(); + if (!input_state.has_input()) { + return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; + } + if (effective_degradation_preference_ == DegradationPreference::DISABLED || + !HasSufficientInputForAdaptation(input_state)) { + return ResourceListenerResponse::kNothing; + } + // Update video input states and encoder settings for accurate adaptation. + stream_adapter_->SetInput(input_state); + // How can this stream be adapted up? + Adaptation adaptation = stream_adapter_->GetAdaptationDown(); + if (adaptation.min_pixel_limit_reached()) + encoder_stats_observer_->OnMinPixelLimitReached(); + if (adaptation.status() != Adaptation::Status::kValid) + return ResourceListenerResponse::kNothing; + // Apply adaptation. + ResourceListenerResponse response = + stream_adapter_->ApplyAdaptation(adaptation); + // Update VideoSourceRestrictions based on adaptation. This also informs the + // |adaptation_listeners_|. + MaybeUpdateVideoSourceRestrictions(&reason_resource); + return response; +} + +void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize( + const Resource& reason_resource) { + VideoAdaptationCounters counters_before = + stream_adapter_->adaptation_counters(); + OnResourceOveruse(reason_resource); + if (degradation_preference_ == DegradationPreference::BALANCED && + stream_adapter_->adaptation_counters().fps_adaptations > + counters_before.fps_adaptations) { + // Oops, we adapted frame rate. Adapt again, maybe it will adapt resolution! + // Though this is not guaranteed... + OnResourceOveruse(reason_resource); + } + if (stream_adapter_->adaptation_counters().resolution_adaptations > + counters_before.resolution_adaptations) { + encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); + } +} + +} // namespace webrtc
diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h new file mode 100644 index 0000000..e877c82 --- /dev/null +++ b/call/adaptation/resource_adaptation_processor.h
@@ -0,0 +1,100 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ +#define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ + +#include <memory> +#include <vector> + +#include "absl/types/optional.h" +#include "api/rtp_parameters.h" +#include "api/video/video_frame.h" +#include "api/video/video_stream_encoder_observer.h" +#include "call/adaptation/resource.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_adapter.h" +#include "call/adaptation/video_stream_input_state.h" +#include "call/adaptation/video_stream_input_state_provider.h" + +namespace webrtc { + +class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, + public ResourceListener { + public: + ResourceAdaptationProcessor( + VideoStreamInputStateProvider* input_state_provider, + VideoStreamEncoderObserver* encoder_stats_observer); + ~ResourceAdaptationProcessor() override; + + // ResourceAdaptationProcessorInterface implementation. + DegradationPreference degradation_preference() const override; + DegradationPreference effective_degradation_preference() const override; + + void StartResourceAdaptation() override; + void StopResourceAdaptation() override; + void AddAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) override; + void AddResource(Resource* resource) override; + + void SetDegradationPreference( + DegradationPreference degradation_preference) override; + void SetIsScreenshare(bool is_screenshare) override; + void ResetVideoSourceRestrictions() override; + + // ResourceListener implementation. + // Triggers OnResourceUnderuse() or OnResourceOveruse(). + ResourceListenerResponse OnResourceUsageStateMeasured( + const Resource& resource) override; + + // May trigger 1-2 adaptations. It is meant to reduce resolution - useful if a + // frame was dropped due to its size - but if you look at the implementation + // you'll find that this is not guaranteed. It could adapt frame rate, which + // does not address the issue. + // TODO(hbos): Can we replace this by something which actually satisfies the + // resolution constraints, or get rid of it altogether? + void TriggerAdaptationDueToFrameDroppedDueToSize( + const Resource& reason_resource); + + private: + bool HasSufficientInputForAdaptation( + const VideoStreamInputState& input_state) const; + + // Performs the adaptation by getting the next target, applying it and + // informing listeners of the new VideoSourceRestriction and adaptation + // counters. + void OnResourceUnderuse(const Resource& reason_resource); + ResourceListenerResponse OnResourceOveruse(const Resource& reason_resource); + + // Needs to be invoked any time |degradation_preference_| or |is_screenshare_| + // changes to ensure |effective_degradation_preference_| is up-to-date. + void MaybeUpdateEffectiveDegradationPreference(); + // If the filtered source restrictions are different than + // |last_reported_source_restrictions_|, inform the listeners. + void MaybeUpdateVideoSourceRestrictions(const Resource* reason); + + // Input and output. + VideoStreamInputStateProvider* const input_state_provider_; + VideoStreamEncoderObserver* const encoder_stats_observer_; + std::vector<ResourceAdaptationProcessorListener*> adaptation_listeners_; + std::vector<Resource*> resources_; + // Adaptation strategy settings. + DegradationPreference degradation_preference_; + DegradationPreference effective_degradation_preference_; + bool is_screenshare_; + // Responsible for generating and applying possible adaptations. + const std::unique_ptr<VideoStreamAdapter> stream_adapter_; + VideoSourceRestrictions last_reported_source_restrictions_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_
diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h index 5d84ffa..89ad903 100644 --- a/call/adaptation/resource_adaptation_processor_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h
@@ -43,13 +43,29 @@ public: virtual ~ResourceAdaptationProcessorInterface(); - virtual void StartResourceAdaptation( - ResourceAdaptationProcessorListener* adaptation_listener) = 0; + virtual DegradationPreference degradation_preference() const = 0; + // Reinterprets "balanced + screenshare" as "maintain-resolution". + // TODO(hbos): Don't do this. This is not what "balanced" means. If the + // application wants to maintain resolution it should set that degradation + // preference rather than depend on non-standard behaviors. + virtual DegradationPreference effective_degradation_preference() const = 0; + + // Starts or stops listening to resources, effectively enabling or disabling + // processing. + // TODO(https://crbug.com/webrtc/11172): Automatically register and unregister + // with AddResource() and RemoveResource() instead. When the processor is + // multi-stream aware, stream-specific resouces will get added and removed + // over time. + virtual void StartResourceAdaptation() = 0; virtual void StopResourceAdaptation() = 0; - // The resource must out-live the module. + virtual void AddAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) = 0; virtual void AddResource(Resource* resource) = 0; + virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; + virtual void SetIsScreenshare(bool is_screenshare) = 0; + virtual void ResetVideoSourceRestrictions() = 0; }; } // namespace webrtc
diff --git a/video/adaptation/encode_usage_resource.h b/video/adaptation/encode_usage_resource.h index 6c1827d..a41211e 100644 --- a/video/adaptation/encode_usage_resource.h +++ b/video/adaptation/encode_usage_resource.h
@@ -35,6 +35,8 @@ explicit EncodeUsageResource( std::unique_ptr<OveruseFrameDetector> overuse_detector); + bool is_started() const { return is_started_; } + void StartCheckForOveruse(CpuOveruseOptions options); void StopCheckForOveruse();
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index f219d37..d2a695e 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -157,7 +157,7 @@ // have this catch-all reason- and stats-based approach. int num_downgrades = FilterVideoAdaptationCountersByDegradationPreference( manager_->active_counts_[reason], - manager_->effective_degradation_preference()) + manager_->effective_degradation_preference_) .Total(); RTC_DCHECK_GE(num_downgrades, 0); return num_downgrades > 0; @@ -219,9 +219,9 @@ // TODO(hbos): Why are we allowing violating balanced settings if adapting due // CPU? Shouldn't this condition be checked regardless of reason? if (reason == VideoAdaptationReason::kQuality && - manager_->effective_degradation_preference() == + manager_->effective_degradation_preference_ == DegradationPreference::BALANCED && - !manager_->stream_adapter_->balanced_settings().CanAdaptUp( + !manager_->balanced_settings_.CanAdaptUp( input_state.video_codec_type(), input_state.frame_size_pixels().value(), manager_->encoder_target_bitrate_bps_.value_or(0))) { @@ -229,7 +229,7 @@ } if (reason == VideoAdaptationReason::kQuality && DidIncreaseResolution(restrictions_before, restrictions_after) && - !manager_->stream_adapter_->balanced_settings().CanAdaptUpResolution( + !manager_->balanced_settings_.CanAdaptUpResolution( input_state.video_codec_type(), input_state.frame_size_pixels().value(), manager_->encoder_target_bitrate_bps_.value_or(0))) { @@ -240,24 +240,24 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, + ResourceAdaptationProcessor* adaptation_processor, + VideoStreamEncoderObserver* encoder_stats_observer, Clock* clock, bool experiment_cpu_load_estimator, - std::unique_ptr<OveruseFrameDetector> overuse_detector, - VideoStreamEncoderObserver* encoder_stats_observer, - ResourceAdaptationProcessorListener* adaptation_listener) + std::unique_ptr<OveruseFrameDetector> overuse_detector) : prevent_adapt_up_due_to_active_counts_(this), prevent_increase_resolution_due_to_bitrate_resource_(this), prevent_adapt_up_in_balanced_resource_(this), encode_usage_resource_(std::move(overuse_detector)), quality_scaler_resource_(), input_state_provider_(input_state_provider), - adaptation_listener_(adaptation_listener), - clock_(clock), - state_(State::kStopped), - experiment_cpu_load_estimator_(experiment_cpu_load_estimator), + adaptation_processor_(adaptation_processor), + encoder_stats_observer_(encoder_stats_observer), degradation_preference_(DegradationPreference::DISABLED), effective_degradation_preference_(DegradationPreference::DISABLED), - stream_adapter_(std::make_unique<VideoStreamAdapter>()), + video_source_restrictions_(), + clock_(clock), + experiment_cpu_load_estimator_(experiment_cpu_load_estimator), initial_frame_dropper_( std::make_unique<InitialFrameDropper>(&quality_scaler_resource_)), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), @@ -265,53 +265,43 @@ quality_rampup_done_(false), quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), encoder_settings_(absl::nullopt), - encoder_stats_observer_(encoder_stats_observer), active_counts_() { - RTC_DCHECK(adaptation_listener_); RTC_DCHECK(encoder_stats_observer_); - AddResource(&prevent_adapt_up_due_to_active_counts_, - VideoAdaptationReason::kQuality); - AddResource(&prevent_increase_resolution_due_to_bitrate_resource_, - VideoAdaptationReason::kQuality); - AddResource(&prevent_adapt_up_in_balanced_resource_, - VideoAdaptationReason::kQuality); - AddResource(&encode_usage_resource_, VideoAdaptationReason::kCpu); - AddResource(&quality_scaler_resource_, VideoAdaptationReason::kQuality); + MapResourceToReason(&prevent_adapt_up_due_to_active_counts_, + VideoAdaptationReason::kQuality); + MapResourceToReason(&prevent_increase_resolution_due_to_bitrate_resource_, + VideoAdaptationReason::kQuality); + MapResourceToReason(&prevent_adapt_up_in_balanced_resource_, + VideoAdaptationReason::kQuality); + MapResourceToReason(&encode_usage_resource_, VideoAdaptationReason::kCpu); + MapResourceToReason(&quality_scaler_resource_, + VideoAdaptationReason::kQuality); } VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() { - RTC_DCHECK_EQ(state_, State::kStopped); + RTC_DCHECK(!encode_usage_resource_.is_started()); } -void VideoStreamEncoderResourceManager::StartResourceAdaptation( - ResourceAdaptationProcessorListener* adaptation_listener) { - RTC_DCHECK_EQ(state_, State::kStopped); +void VideoStreamEncoderResourceManager::SetDegradationPreferences( + DegradationPreference degradation_preference, + DegradationPreference effective_degradation_preference) { + degradation_preference_ = degradation_preference; + effective_degradation_preference_ = effective_degradation_preference; + UpdateStatsAdaptationSettings(); +} + +void VideoStreamEncoderResourceManager::StartEncodeUsageResource() { + RTC_DCHECK(!encode_usage_resource_.is_started()); RTC_DCHECK(encoder_settings_.has_value()); - // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener - // should be passed in and why. If resources are separated from modules then - // those resources may be started or stopped separately from the module. - RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); encode_usage_resource_.StartCheckForOveruse(GetCpuOveruseOptions()); - for (auto& resource_and_reason : resources_) { - resource_and_reason.resource->RegisterListener(this); - } - state_ = State::kStarted; } -void VideoStreamEncoderResourceManager::StopResourceAdaptation() { +void VideoStreamEncoderResourceManager::StopManagedResources() { encode_usage_resource_.StopCheckForOveruse(); quality_scaler_resource_.StopCheckForOveruse(); - for (auto& resource_and_reason : resources_) { - resource_and_reason.resource->UnregisterListener(this); - } - state_ = State::kStopped; } -void VideoStreamEncoderResourceManager::AddResource(Resource* resource) { - return AddResource(resource, VideoAdaptationReason::kCpu); -} - -void VideoStreamEncoderResourceManager::AddResource( +void VideoStreamEncoderResourceManager::MapResourceToReason( Resource* resource, VideoAdaptationReason reason) { RTC_DCHECK(resource); @@ -323,17 +313,18 @@ resources_.emplace_back(resource, reason); } -void VideoStreamEncoderResourceManager::SetDegradationPreference( - DegradationPreference degradation_preference) { - degradation_preference_ = degradation_preference; - UpdateStatsAdaptationSettings(); - MaybeUpdateEffectiveDegradationPreference(); +std::vector<Resource*> VideoStreamEncoderResourceManager::MappedResources() + const { + std::vector<Resource*> resources; + for (auto const& resource_and_reason : resources_) { + resources.push_back(resource_and_reason.resource); + } + return resources; } void VideoStreamEncoderResourceManager::SetEncoderSettings( EncoderSettings encoder_settings) { encoder_settings_ = std::move(encoder_settings); - MaybeUpdateEffectiveDegradationPreference(); quality_rampup_experiment_.SetMaxBitrate( LastInputFrameSizeOrDefault(), @@ -362,25 +353,9 @@ encoder_rates_ = encoder_rates; } -void VideoStreamEncoderResourceManager::ResetVideoSourceRestrictions() { - stream_adapter_->ClearRestrictions(); - MaybeUpdateVideoSourceRestrictions(nullptr); -} - void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() { - VideoAdaptationCounters counters_before = - stream_adapter_->adaptation_counters(); - OnResourceOveruse(quality_scaler_resource_); - if (degradation_preference_ == DegradationPreference::BALANCED && - stream_adapter_->adaptation_counters().fps_adaptations > - counters_before.fps_adaptations) { - // Adapt framerate in same step as resolution. - OnResourceOveruse(quality_scaler_resource_); - } - if (stream_adapter_->adaptation_counters().resolution_adaptations > - counters_before.resolution_adaptations) { - encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); - } + adaptation_processor_->TriggerAdaptationDueToFrameDroppedDueToSize( + quality_scaler_resource_); initial_frame_dropper_->OnFrameDroppedDueToSize(); } @@ -461,7 +436,7 @@ if (degradation_preference_ == DegradationPreference::BALANCED && quality_scaler_resource_.is_started()) { absl::optional<VideoEncoder::QpThresholds> thresholds = - stream_adapter_->balanced_settings().GetQpThresholds( + balanced_settings_.GetQpThresholds( GetVideoCodecTypeOrGeneric(encoder_settings_), LastInputFrameSizeOrDefault()); if (thresholds) { @@ -471,30 +446,6 @@ UpdateStatsAdaptationSettings(); } -ResourceListenerResponse -VideoStreamEncoderResourceManager::OnResourceUsageStateMeasured( - const Resource& resource) { - switch (resource.usage_state()) { - case ResourceUsageState::kOveruse: - return OnResourceOveruse(resource); - case ResourceUsageState::kStable: - // Do nothing. - // TODO(https://crbug.com/webrtc/11172): Delete kStable in favor of null. - return ResourceListenerResponse::kNothing; - case ResourceUsageState::kUnderuse: - OnResourceUnderuse(resource); - return ResourceListenerResponse::kNothing; - } -} - -bool VideoStreamEncoderResourceManager::HasSufficientInputForAdaptation( - const VideoStreamInputState& input_state) const { - return input_state.HasInputFrameSizeAndFramesPerSecond() && - (effective_degradation_preference_ != - DegradationPreference::MAINTAIN_RESOLUTION || - input_state.frames_per_second() >= kMinFrameRateFps); -} - VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource( const Resource& resource) const { const auto& registered_resource = @@ -506,67 +457,6 @@ return registered_resource->reason; } -void VideoStreamEncoderResourceManager::OnResourceUnderuse( - const Resource& reason_resource) { - VideoStreamInputState input_state = input_state_provider_->InputState(); - if (effective_degradation_preference_ == DegradationPreference::DISABLED || - !HasSufficientInputForAdaptation(input_state)) { - return; - } - // Update video input states and encoder settings for accurate adaptation. - stream_adapter_->SetInput(input_state); - // How can this stream be adapted up? - Adaptation adaptation = stream_adapter_->GetAdaptationUp(); - if (adaptation.status() != Adaptation::Status::kValid) - return; - // Are all resources OK with this adaptation being applied? - VideoSourceRestrictions restrictions_before = - stream_adapter_->source_restrictions(); - VideoSourceRestrictions restrictions_after = - stream_adapter_->PeekNextRestrictions(adaptation); - if (!absl::c_all_of(resources_, [&input_state, &restrictions_before, - &restrictions_after, &reason_resource]( - ResourceAndReason resource_and_reason) { - return resource_and_reason.resource->IsAdaptationUpAllowed( - input_state, restrictions_before, restrictions_after, - reason_resource); - })) { - return; - } - // Apply adaptation. - stream_adapter_->ApplyAdaptation(adaptation); - // Update VideoSourceRestrictions based on adaptation. This also informs the - // |adaptation_listener_|. - MaybeUpdateVideoSourceRestrictions(&reason_resource); -} - -ResourceListenerResponse VideoStreamEncoderResourceManager::OnResourceOveruse( - const Resource& reason_resource) { - VideoStreamInputState input_state = input_state_provider_->InputState(); - if (!input_state.has_input()) { - return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; - } - if (effective_degradation_preference_ == DegradationPreference::DISABLED || - !HasSufficientInputForAdaptation(input_state)) { - return ResourceListenerResponse::kNothing; - } - // Update video input states and encoder settings for accurate adaptation. - stream_adapter_->SetInput(input_state); - // How can this stream be adapted down? - Adaptation adaptation = stream_adapter_->GetAdaptationDown(); - if (adaptation.min_pixel_limit_reached()) - encoder_stats_observer_->OnMinPixelLimitReached(); - if (adaptation.status() != Adaptation::Status::kValid) - return ResourceListenerResponse::kNothing; - // Apply adaptation. - ResourceListenerResponse response = - stream_adapter_->ApplyAdaptation(adaptation); - // Update VideoSourceRestrictions based on adaptation. This also informs the - // |adaptation_listener_|. - MaybeUpdateVideoSourceRestrictions(&reason_resource); - return response; -} - // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle // pipelining encoders better (multiple input frames before something comes // out). This should effectively turn off CPU adaptations for systems that @@ -594,43 +484,11 @@ kDefaultInputPixelsWidth * kDefaultInputPixelsHeight); } -void VideoStreamEncoderResourceManager:: - MaybeUpdateEffectiveDegradationPreference() { - bool is_screenshare = encoder_settings_.has_value() && - encoder_settings_->encoder_config().content_type == - VideoEncoderConfig::ContentType::kScreen; - effective_degradation_preference_ = - (is_screenshare && - degradation_preference_ == DegradationPreference::BALANCED) - ? DegradationPreference::MAINTAIN_RESOLUTION - : degradation_preference_; - stream_adapter_->SetDegradationPreference(effective_degradation_preference_); - MaybeUpdateVideoSourceRestrictions(nullptr); -} - -void VideoStreamEncoderResourceManager::MaybeUpdateVideoSourceRestrictions( - const Resource* reason_resource) { - VideoSourceRestrictions new_restrictions = - FilterRestrictionsByDegradationPreference( - stream_adapter_->source_restrictions(), degradation_preference_); - if (video_source_restrictions_ != new_restrictions) { - video_source_restrictions_ = std::move(new_restrictions); - // TODO(https://crbug.com/webrtc/11172): Support multiple listeners and - // loop through them here instead of calling two hardcoded listeners (|this| - // and |adaptation_listener_|). - OnVideoSourceRestrictionsUpdated(video_source_restrictions_, - stream_adapter_->adaptation_counters(), - reason_resource); - adaptation_listener_->OnVideoSourceRestrictionsUpdated( - video_source_restrictions_, stream_adapter_->adaptation_counters(), - reason_resource); - } -} - void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, const Resource* reason) { + video_source_restrictions_ = restrictions; VideoAdaptationCounters previous_adaptation_counters = active_counts_[VideoAdaptationReason::kQuality] + active_counts_[VideoAdaptationReason::kCpu]; @@ -798,7 +656,7 @@ if (try_quality_rampup && qp_counts.resolution_adaptations > 0 && cpu_counts.Total() == 0) { RTC_LOG(LS_INFO) << "Reset quality limitations."; - ResetVideoSourceRestrictions(); + adaptation_processor_->ResetVideoSourceRestrictions(); quality_rampup_done_ = true; } }
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index d99b6ec..d07c31b 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -29,6 +29,7 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_config.h" #include "call/adaptation/resource.h" +#include "call/adaptation/resource_adaptation_processor.h" #include "call/adaptation/resource_adaptation_processor_interface.h" #include "call/adaptation/video_stream_adapter.h" #include "call/adaptation/video_stream_input_state_provider.h" @@ -48,48 +49,35 @@ extern const int kDefaultInputPixelsWidth; extern const int kDefaultInputPixelsHeight; -// This class is used by the VideoStreamEncoder and is responsible for adapting -// resolution up or down based on encode usage percent. It keeps track of video -// source settings, adaptation counters and may get influenced by -// VideoStreamEncoder's quality scaler through AdaptUp() and AdaptDown() calls. +// Owns adaptation-related Resources pertaining to a single VideoStreamEncoder +// and passes on the relevant input from the encoder to the resources. The +// resources provide resource usage states to the ResourceAdaptationProcessor +// which is responsible for reconfiguring streams in order not to overuse +// resources. // -// This class is single-threaded. The caller is responsible for ensuring safe -// usage. -// TODO(hbos): Add unittests specific to this class, it is currently only tested -// indirectly in video_stream_encoder_unittest.cc and other tests exercising -// VideoStreamEncoder. +// The manager is also involved with various mitigations not part of the +// ResourceAdaptationProcessor code such as the inital frame dropping. class VideoStreamEncoderResourceManager - : public ResourceAdaptationProcessorInterface, - public ResourceListener, - public ResourceAdaptationProcessorListener { + : public ResourceAdaptationProcessorListener { public: - // The processor can be constructed on any sequence, but must be initialized - // and used on a single sequence, e.g. the encoder queue. VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, + ResourceAdaptationProcessor* adaptation_processor, + VideoStreamEncoderObserver* encoder_stats_observer, Clock* clock, bool experiment_cpu_load_estimator, - std::unique_ptr<OveruseFrameDetector> overuse_detector, - VideoStreamEncoderObserver* encoder_stats_observer, - ResourceAdaptationProcessorListener* adaptation_listener); + std::unique_ptr<OveruseFrameDetector> overuse_detector); ~VideoStreamEncoderResourceManager() override; - DegradationPreference degradation_preference() const { - return degradation_preference_; - } - DegradationPreference effective_degradation_preference() const { - return effective_degradation_preference_; - } + void SetDegradationPreferences( + DegradationPreference degradation_preference, + DegradationPreference effective_degradation_preference); - // ResourceAdaptationProcessorInterface implementation. - void StartResourceAdaptation( - ResourceAdaptationProcessorListener* adaptation_listener) override; - void StopResourceAdaptation() override; - // Uses a default VideoAdaptationReason of kCpu. - void AddResource(Resource* resource) override; - void AddResource(Resource* resource, VideoAdaptationReason reason); - void SetDegradationPreference( - DegradationPreference degradation_preference) override; + // Starts the encode usage resource. The quality scaler resource is + // automatically started on being configured. + void StartEncodeUsageResource(); + // Stops the encode usage and quality scaler resources if not already stopped. + void StopManagedResources(); // Settings that affect the VideoStreamEncoder-specific resources. void SetEncoderSettings(EncoderSettings encoder_settings); @@ -110,14 +98,20 @@ int64_t time_sent_in_us, absl::optional<int> encode_duration_us); void OnFrameDropped(EncodedImageCallback::DropReason reason); + + // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order + // to be able to update |active_counts_|, which is used... + // - Legacy getStats() purposes. + // - Preventing adapting up in some circumstances (which may be questionable). + // TODO(hbos): Can we get rid of this? + void MapResourceToReason(Resource* resource, VideoAdaptationReason reason); + std::vector<Resource*> MappedResources() const; // If true, the VideoStreamEncoder should eexecute its logic to maybe drop // frames baseed on size and bitrate. bool DropInitialFrames() const; - // ResourceUsageListener implementation. - ResourceListenerResponse OnResourceUsageStateMeasured( - const Resource& resource) override; - + // ResourceAdaptationProcessorListener implementation. + // Updates |video_source_restrictions_| and |active_counts_|. void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, @@ -138,10 +132,6 @@ private: class InitialFrameDropper; - enum class State { kStopped, kStarted }; - - bool HasSufficientInputForAdaptation( - const VideoStreamInputState& input_state) const; VideoAdaptationReason GetReasonFromResource(const Resource& resource) const; // Performs the adaptation by getting the next target, applying it and @@ -152,15 +142,6 @@ CpuOveruseOptions GetCpuOveruseOptions() const; int LastInputFrameSizeOrDefault() const; - // Reinterprets "balanced + screenshare" as "maintain-resolution". - // When screensharing, as far as ResourceAdaptationProcessor logic is - // concerned, we ALWAYS use "maintain-resolution". However, on a different - // layer we may cap the video resolution to 720p to make high fps - // screensharing feasible. This means that on the API layer the preference is - // "balanced" (allowing reduction in both resolution and frame rate) but on - // this layer (not responsible for caping to 720p) the preference is the same - // as "maintain-resolution". - void MaybeUpdateEffectiveDegradationPreference(); // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener // to reconfigure the source accordingly. @@ -183,7 +164,6 @@ // TODO(https://crbug.com/webrtc/11222) Move experiment details into an inner // class. void MaybePerformQualityRampupExperiment(); - void ResetVideoSourceRestrictions(); void ResetActiveCounts(); std::string ActiveCountsToString() const; @@ -256,25 +236,23 @@ QualityScalerResource quality_scaler_resource_; VideoStreamInputStateProvider* const input_state_provider_; - ResourceAdaptationProcessorListener* const adaptation_listener_; - Clock* clock_; - State state_; - const bool experiment_cpu_load_estimator_; - // The restrictions that |adaptation_listener_| is informed of. - VideoSourceRestrictions video_source_restrictions_; + ResourceAdaptationProcessor* const adaptation_processor_; + VideoStreamEncoderObserver* const encoder_stats_observer_; + DegradationPreference degradation_preference_; DegradationPreference effective_degradation_preference_; - // Keeps track of source restrictions that this adaptation processor outputs. - const std::unique_ptr<VideoStreamAdapter> stream_adapter_; + VideoSourceRestrictions video_source_restrictions_; + + const BalancedDegradationSettings balanced_settings_; + Clock* clock_; + const bool experiment_cpu_load_estimator_; const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_; const bool quality_scaling_experiment_enabled_; - // This is the last non-zero target bitrate for the encoder. absl::optional<uint32_t> encoder_target_bitrate_bps_; absl::optional<VideoEncoder::RateControlParameters> encoder_rates_; bool quality_rampup_done_; QualityRampupExperiment quality_rampup_experiment_; absl::optional<EncoderSettings> encoder_settings_; - VideoStreamEncoderObserver* const encoder_stats_observer_; // Ties a resource to a reason for statistical reporting. This AdaptReason is // also used by this module to make decisions about how to adapt up/down.
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index d82b240..af97ce8 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc
@@ -255,25 +255,31 @@ automatic_animation_detection_experiment_( ParseAutomatincAnimationDetectionFieldTrial()), encoder_switch_requested_(false), - video_source_sink_controller_(std::make_unique<VideoSourceSinkController>( - /*sink=*/this, - /*source=*/nullptr)), - input_state_provider_(std::make_unique<VideoStreamInputStateProvider>( - encoder_stats_observer)), - stream_resource_manager_( - std::make_unique<VideoStreamEncoderResourceManager>( - input_state_provider_.get(), - clock_, - settings_.experiment_cpu_load_estimator, - std::move(overuse_detector), - encoder_stats_observer, - /*adaptation_listener=*/this)), + input_state_provider_(encoder_stats_observer), + resource_adaptation_processor_(&input_state_provider_, + encoder_stats_observer), + stream_resource_manager_(&input_state_provider_, + &resource_adaptation_processor_, + encoder_stats_observer, + clock_, + settings_.experiment_cpu_load_estimator, + std::move(overuse_detector)), + video_source_sink_controller_(/*sink=*/this, + /*source=*/nullptr), encoder_queue_(task_queue_factory->CreateTaskQueue( "EncoderQueue", TaskQueueFactory::Priority::NORMAL)) { RTC_DCHECK(encoder_stats_observer); RTC_DCHECK_GE(number_of_cores, 1); + resource_adaptation_processor_.AddAdaptationListener( + &stream_resource_manager_); + resource_adaptation_processor_.AddAdaptationListener(this); + + // Add the stream resource manager's resources to the processor. + for (Resource* resource : stream_resource_manager_.MappedResources()) + resource_adaptation_processor_.AddResource(resource); + for (auto& state : encoder_buffer_state_) state.fill(std::numeric_limits<int64_t>::max()); } @@ -286,10 +292,11 @@ void VideoStreamEncoder::Stop() { RTC_DCHECK_RUN_ON(&thread_checker_); - video_source_sink_controller_->SetSource(nullptr); + video_source_sink_controller_.SetSource(nullptr); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); - stream_resource_manager_->StopResourceAdaptation(); + stream_resource_manager_.StopManagedResources(); + resource_adaptation_processor_.StopResourceAdaptation(); rate_allocator_ = nullptr; bitrate_observer_ = nullptr; ReleaseEncoder(); @@ -325,20 +332,25 @@ rtc::VideoSourceInterface<VideoFrame>* source, const DegradationPreference& degradation_preference) { RTC_DCHECK_RUN_ON(&thread_checker_); - video_source_sink_controller_->SetSource(source); + video_source_sink_controller_.SetSource(source); encoder_queue_.PostTask([this, source, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); - input_state_provider_->OnHasInputChanged(source); - stream_resource_manager_->SetDegradationPreference(degradation_preference); - if (encoder_) - stream_resource_manager_->ConfigureQualityScaler( + input_state_provider_.OnHasInputChanged(source); + resource_adaptation_processor_.SetDegradationPreference( + degradation_preference); + stream_resource_manager_.SetDegradationPreferences( + resource_adaptation_processor_.degradation_preference(), + resource_adaptation_processor_.effective_degradation_preference()); + if (encoder_) { + stream_resource_manager_.ConfigureQualityScaler( encoder_->GetEncoderInfo()); + } }); } void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { - video_source_sink_controller_->SetRotationApplied(rotation_applied); - video_source_sink_controller_->PushSourceSinkSettings(); + video_source_sink_controller_.SetRotationApplied(rotation_applied); + video_source_sink_controller_.PushSourceSinkSettings(); encoder_queue_.PostTask([this, sink] { RTC_DCHECK_RUN_ON(&encoder_queue_); sink_ = sink; @@ -351,7 +363,7 @@ encoder_target_bitrate_bps_ = start_bitrate_bps != 0 ? absl::optional<uint32_t>(start_bitrate_bps) : absl::nullopt; - stream_resource_manager_->SetStartBitrate( + stream_resource_manager_.SetStartBitrate( DataRate::BitsPerSec(start_bitrate_bps)); }); } @@ -568,12 +580,11 @@ max_framerate = std::max(stream.max_framerate, max_framerate); } int alignment = encoder_->GetEncoderInfo().requested_resolution_alignment; - if (max_framerate != - video_source_sink_controller_->frame_rate_upper_limit() || - alignment != video_source_sink_controller_->resolution_alignment()) { - video_source_sink_controller_->SetFrameRateUpperLimit(max_framerate); - video_source_sink_controller_->SetResolutionAlignment(alignment); - video_source_sink_controller_->PushSourceSinkSettings(); + if (max_framerate != video_source_sink_controller_.frame_rate_upper_limit() || + alignment != video_source_sink_controller_.resolution_alignment()) { + video_source_sink_controller_.SetFrameRateUpperLimit(max_framerate); + video_source_sink_controller_.SetResolutionAlignment(alignment); + video_source_sink_controller_.PushSourceSinkSettings(); } if (codec.maxBitrate == 0) { @@ -656,8 +667,10 @@ } if (pending_encoder_creation_) { - stream_resource_manager_->StopResourceAdaptation(); - stream_resource_manager_->StartResourceAdaptation(this); + stream_resource_manager_.StopManagedResources(); + resource_adaptation_processor_.StopResourceAdaptation(); + stream_resource_manager_.StartEncodeUsageResource(); + resource_adaptation_processor_.StartResourceAdaptation(); pending_encoder_creation_ = false; } @@ -730,14 +743,19 @@ std::move(streams), is_svc, encoder_config_.content_type, encoder_config_.min_transmit_bitrate_bps); - stream_resource_manager_->ConfigureQualityScaler(info); + stream_resource_manager_.ConfigureQualityScaler(info); } void VideoStreamEncoder::OnEncoderSettingsChanged() { EncoderSettings encoder_settings(encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_); - input_state_provider_->OnEncoderSettingsChanged(encoder_settings); - stream_resource_manager_->SetEncoderSettings(encoder_settings); + resource_adaptation_processor_.SetIsScreenshare( + encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen); + stream_resource_manager_.SetDegradationPreferences( + resource_adaptation_processor_.degradation_preference(), + resource_adaptation_processor_.effective_degradation_preference()); + input_state_provider_.OnEncoderSettingsChanged(encoder_settings); + stream_resource_manager_.SetEncoderSettings(encoder_settings); } void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { @@ -974,14 +992,14 @@ frame_encode_metadata_writer_.OnSetRates( rate_settings.rate_control.bitrate, static_cast<uint32_t>(rate_settings.rate_control.framerate_fps + 0.5)); - stream_resource_manager_->SetEncoderRates(rate_settings.rate_control); + stream_resource_manager_.SetEncoderRates(rate_settings.rate_control); } } void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, int64_t time_when_posted_us) { RTC_DCHECK_RUN_ON(&encoder_queue_); - input_state_provider_->OnFrameSizeObserved(video_frame.size()); + input_state_provider_.OnFrameSizeObserved(video_frame.size()); if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || video_frame.height() != last_frame_info_->height || @@ -1043,7 +1061,7 @@ if (DropDueToSize(video_frame.size())) { RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; - stream_resource_manager_->OnFrameDroppedDueToSize(); + stream_resource_manager_.OnFrameDroppedDueToSize(); // Storing references to a native buffer risks blocking frame capture. if (video_frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) { @@ -1057,7 +1075,7 @@ } return; } - stream_resource_manager_->OnMaybeEncodeFrame(); + stream_resource_manager_.OnMaybeEncodeFrame(); if (EncoderPaused()) { // Storing references to a native buffer risks blocking frame capture. @@ -1240,7 +1258,7 @@ TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); - stream_resource_manager_->OnEncodeStarted(out_frame, time_when_posted_us); + stream_resource_manager_.OnEncodeStarted(out_frame, time_when_posted_us); RTC_DCHECK_LE(send_codec_.width, out_frame.width()); RTC_DCHECK_LE(send_codec_.height, out_frame.height()); @@ -1515,7 +1533,7 @@ sink_->OnDroppedFrame(reason); encoder_queue_.PostTask([this, reason] { RTC_DCHECK_RUN_ON(&encoder_queue_); - stream_resource_manager_->OnFrameDropped(reason); + stream_resource_manager_.OnFrameDropped(reason); }); } @@ -1614,7 +1632,7 @@ if (target_bitrate.bps() != 0) encoder_target_bitrate_bps_ = target_bitrate.bps(); - stream_resource_manager_->SetTargetBitrate(target_bitrate); + stream_resource_manager_.SetTargetBitrate(target_bitrate); if (video_suspension_changed) { RTC_LOG(LS_INFO) << "Video suspend state changed to: " @@ -1631,7 +1649,7 @@ } bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { - if (!stream_resource_manager_->DropInitialFrames() || + if (!stream_resource_manager_.DropInitialFrames() || !encoder_target_bitrate_bps_.has_value()) { return false; } @@ -1659,8 +1677,8 @@ const VideoAdaptationCounters& adaptation_counters, const Resource* reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); - video_source_sink_controller_->SetRestrictions(std::move(restrictions)); - video_source_sink_controller_->PushSourceSinkSettings(); + video_source_sink_controller_.SetRestrictions(std::move(restrictions)); + video_source_sink_controller_.PushSourceSinkSettings(); } void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image, @@ -1705,8 +1723,8 @@ } } - stream_resource_manager_->OnEncodeCompleted(encoded_image, time_sent_us, - encode_duration_us); + stream_resource_manager_.OnEncodeCompleted(encoded_image, time_sent_us, + encode_duration_us); if (bitrate_adjuster_) { bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index); } @@ -1861,7 +1879,7 @@ if (!automatic_animation_detection_experiment_.enabled || encoder_config_.content_type != VideoEncoderConfig::ContentType::kScreen || - stream_resource_manager_->degradation_preference() != + resource_adaptation_processor_.degradation_preference() != DegradationPreference::BALANCED) { return; } @@ -1919,16 +1937,17 @@ RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent " "animation detection."; } - video_source_sink_controller_->SetPixelsPerFrameUpperLimit( + video_source_sink_controller_.SetPixelsPerFrameUpperLimit( should_cap_resolution ? absl::optional<size_t>(kMaxAnimationPixels) : absl::nullopt); - video_source_sink_controller_->PushSourceSinkSettings(); + video_source_sink_controller_.PushSourceSinkSettings(); } } void VideoStreamEncoder::InjectAdaptationResource( Resource* resource, VideoAdaptationReason reason) { - stream_resource_manager_->AddResource(resource, reason); + stream_resource_manager_.MapResourceToReason(resource, reason); + resource_adaptation_processor_.AddResource(resource); } } // namespace webrtc
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 4ad7edc..f593cf2 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h
@@ -397,23 +397,29 @@ // track of whether a request has been made or not. bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_); - // The controller updates the sink wants based on restrictions that come from - // the resource adaptation processor or adaptation due to bandwidth - // adaptation. + // Provies video stream input states: current resolution and frame rate. + VideoStreamInputStateProvider input_state_provider_ + RTC_GUARDED_BY(&encoder_queue_); + // Responsible for adapting input resolution or frame rate to ensure resources + // (e.g. CPU or bandwidth) are not overused. + ResourceAdaptationProcessor resource_adaptation_processor_ + RTC_GUARDED_BY(&encoder_queue_); + // Handles input, output and stats reporting related to VideoStreamEncoder + // specific resources, such as "encode usage percent" measurements and "QP + // scaling". Also involved with various mitigations such as inital frame + // dropping. + VideoStreamEncoderResourceManager stream_resource_manager_ + RTC_GUARDED_BY(&encoder_queue_); + // Carries out the VideoSourceRestrictions provided by the + // ResourceAdaptationProcessor, i.e. reconfigures the source of video frames + // to provide us with different resolution or frame rate. // - // This is used on the encoder queue, with a few exceptions: + // Used on the |encoder_queue_| with a few exceptions: // - VideoStreamEncoder::SetSource() invokes SetSource(). // - VideoStreamEncoder::SetSink() invokes SetRotationApplied() and // PushSourceSinkSettings(). // - VideoStreamEncoder::Stop() invokes SetSource(). - // TODO(hbos): If these can be moved to the encoder queue, - // VideoSourceSinkController can be made single-threaded, and its lock can be - // replaced with a sequence checker. - std::unique_ptr<VideoSourceSinkController> video_source_sink_controller_; - std::unique_ptr<VideoStreamInputStateProvider> input_state_provider_ - RTC_GUARDED_BY(&encoder_queue_); - std::unique_ptr<VideoStreamEncoderResourceManager> stream_resource_manager_ - RTC_GUARDED_BY(&encoder_queue_); + VideoSourceSinkController video_source_sink_controller_; // All public methods are proxied to |encoder_queue_|. It must must be // destroyed first to make sure no tasks are run that use other members.