blob: 32e260d45924270d282483902cf034461c2c02e8 [file] [log] [blame]
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_
#define VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_
#include <atomic>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "api/adaptation/resource.h"
#include "api/field_trials_view.h"
#include "api/rtp_parameters.h"
#include "api/scoped_refptr.h"
#include "api/task_queue/task_queue_base.h"
#include "api/video/video_adaptation_counters.h"
#include "api/video/video_adaptation_reason.h"
#include "api/video/video_frame.h"
#include "api/video/video_source_interface.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource_adaptation_processor_interface.h"
#include "call/adaptation/video_stream_adapter.h"
#include "call/adaptation/video_stream_input_state_provider.h"
#include "rtc_base/experiments/quality_scaler_settings.h"
#include "rtc_base/ref_count.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/thread_annotations.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/balanced_constraint.h"
#include "video/adaptation/bandwidth_quality_scaler_resource.h"
#include "video/adaptation/bitrate_constraint.h"
#include "video/adaptation/encode_usage_resource.h"
#include "video/adaptation/overuse_frame_detector.h"
#include "video/adaptation/pixel_limit_resource.h"
#include "video/adaptation/quality_scaler_resource.h"
#include "video/adaptation/video_stream_encoder_resource.h"
#include "video/config/video_encoder_config.h"
#include "video/video_stream_encoder_observer.h"
namespace webrtc {
// The assumed input frame size if we have not yet received a frame.
// TODO(hbos): This is 144p - why are we assuming super low quality? Seems like
// a bad heuristic.
extern const int kDefaultInputPixelsWidth;
extern const int kDefaultInputPixelsHeight;
// Owns adaptation-related Resources pertaining to a single VideoStreamEncoder
// and passes on the relevant input from the encoder to the resources. The
// resources provide resource usage states to the ResourceAdaptationProcessor
// which is responsible for reconfiguring streams in order not to overuse
// resources.
//
// The manager is also involved with various mitigations not part of the
// ResourceAdaptationProcessor code such as the initial frame dropping.
class VideoStreamEncoderResourceManager
: public VideoSourceRestrictionsListener,
public ResourceLimitationsListener {
public:
VideoStreamEncoderResourceManager(
VideoStreamInputStateProvider* input_state_provider,
VideoStreamEncoderObserver* encoder_stats_observer,
Clock* clock,
bool experiment_cpu_load_estimator,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
DegradationPreferenceProvider* degradation_preference_provider,
const FieldTrialsView& field_trials);
~VideoStreamEncoderResourceManager() override;
void Initialize(TaskQueueBase* encoder_queue);
void SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor,
VideoStreamAdapter* stream_adapter);
// TODO(https://crbug.com/webrtc/11563): The degradation preference is a
// setting of the Processor, it does not belong to the Manager - can we get
// rid of this?
void SetDegradationPreferences(DegradationPreference degradation_preference);
DegradationPreference degradation_preference() const;
void ConfigureEncodeUsageResource();
// Initializes the pixel limit resource if the "WebRTC-PixelLimitResource"
// field trial is enabled. This can be used for testing.
void MaybeInitializePixelLimitResource();
// Stops the encode usage and quality scaler resources if not already stopped.
// If the pixel limit resource was created it is also stopped and nulled.
void StopManagedResources();
// Settings that affect the VideoStreamEncoder-specific resources.
void SetEncoderSettings(EncoderSettings encoder_settings);
void SetStartBitrate(DataRate start_bitrate);
void SetTargetBitrate(DataRate target_bitrate);
void SetEncoderRates(
const VideoEncoder::RateControlParameters& encoder_rates);
// TODO(https://crbug.com/webrtc/11338): This can be made private if we
// configure on SetDegredationPreference and SetEncoderSettings.
void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
void ConfigureBandwidthQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info);
// Methods corresponding to different points in the encoding pipeline.
void OnFrameDroppedDueToSize();
void OnMaybeEncodeFrame();
void OnEncodeStarted(const VideoFrame& cropped_frame,
int64_t time_when_first_seen_us);
void OnEncodeCompleted(const EncodedImage& encoded_image,
int64_t time_sent_in_us,
std::optional<int> encode_duration_us,
DataSize frame_size);
void OnFrameDropped(EncodedImageCallback::DropReason reason);
// Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order
// to update legacy getStats().
void AddResource(rtc::scoped_refptr<Resource> resource,
VideoAdaptationReason reason);
void RemoveResource(rtc::scoped_refptr<Resource> resource);
std::vector<AdaptationConstraint*> AdaptationConstraints() const;
// If true, the VideoStreamEncoder should execute its logic to maybe drop
// frames based on size and bitrate.
bool DropInitialFrames() const;
std::optional<uint32_t> SingleActiveStreamPixels() const;
std::optional<uint32_t> UseBandwidthAllocationBps() const;
// VideoSourceRestrictionsListener implementation.
// Updates `video_source_restrictions_`.
void OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
rtc::scoped_refptr<Resource> reason,
const VideoSourceRestrictions& unfiltered_restrictions) override;
void OnResourceLimitationChanged(
rtc::scoped_refptr<Resource> resource,
const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
resource_limitations) override;
static bool IsSimulcastOrMultipleSpatialLayers(
const VideoEncoderConfig& encoder_config,
const VideoCodec& video_codec);
private:
class InitialFrameDropper;
VideoAdaptationReason GetReasonFromResource(
rtc::scoped_refptr<Resource> resource) const;
CpuOveruseOptions GetCpuOveruseOptions() const;
int LastFrameSizeOrDefault() const;
// Calculates an up-to-date value of the target frame rate and informs the
// `encode_usage_resource_` of the new value.
void MaybeUpdateTargetFrameRate();
// Use nullopt to disable quality scaling.
void UpdateQualityScalerSettings(
std::optional<VideoEncoder::QpThresholds> qp_thresholds);
void UpdateBandwidthQualityScalerSettings(
bool bandwidth_quality_scaling_allowed,
const std::vector<VideoEncoder::ResolutionBitrateLimits>&
resolution_bitrate_limits);
void UpdateStatsAdaptationSettings() const;
static std::string ActiveCountsToString(
const std::map<VideoAdaptationReason, VideoAdaptationCounters>&
active_counts);
const FieldTrialsView& field_trials_;
DegradationPreferenceProvider* const degradation_preference_provider_;
std::unique_ptr<BitrateConstraint> bitrate_constraint_
RTC_GUARDED_BY(encoder_queue_);
const std::unique_ptr<BalancedConstraint> balanced_constraint_
RTC_GUARDED_BY(encoder_queue_);
const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;
const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource_;
const rtc::scoped_refptr<BandwidthQualityScalerResource>
bandwidth_quality_scaler_resource_;
TaskQueueBase* encoder_queue_;
VideoStreamInputStateProvider* const input_state_provider_
RTC_GUARDED_BY(encoder_queue_);
ResourceAdaptationProcessorInterface* adaptation_processor_;
VideoStreamAdapter* stream_adapter_ RTC_GUARDED_BY(encoder_queue_);
// Thread-safe.
VideoStreamEncoderObserver* const encoder_stats_observer_;
DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_);
VideoSourceRestrictions video_source_restrictions_
RTC_GUARDED_BY(encoder_queue_);
VideoAdaptationCounters current_adaptation_counters_
RTC_GUARDED_BY(encoder_queue_);
const BalancedDegradationSettings balanced_settings_;
Clock* clock_ RTC_GUARDED_BY(encoder_queue_);
const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_);
const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_
RTC_GUARDED_BY(encoder_queue_);
const bool quality_scaling_experiment_enabled_ RTC_GUARDED_BY(encoder_queue_);
const bool pixel_limit_resource_experiment_enabled_
RTC_GUARDED_BY(encoder_queue_);
std::optional<uint32_t> encoder_target_bitrate_bps_
RTC_GUARDED_BY(encoder_queue_);
std::optional<VideoEncoder::RateControlParameters> encoder_rates_
RTC_GUARDED_BY(encoder_queue_);
std::optional<EncoderSettings> encoder_settings_
RTC_GUARDED_BY(encoder_queue_);
// Ties a resource to a reason for statistical reporting. This AdaptReason is
// also used by this module to make decisions about how to adapt up/down.
std::map<rtc::scoped_refptr<Resource>, VideoAdaptationReason> resources_
RTC_GUARDED_BY(encoder_queue_);
};
} // namespace webrtc
#endif // VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_