| /* |
| * Copyright 2020 The WebRTC Project Authors. All rights reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #ifndef CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ |
| #define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ |
| |
| #include <memory> |
| #include <utility> |
| #include <vector> |
| |
| #include "absl/types/optional.h" |
| #include "absl/types/variant.h" |
| #include "api/adaptation/resource.h" |
| #include "api/field_trials_view.h" |
| #include "api/rtp_parameters.h" |
| #include "api/video/video_adaptation_counters.h" |
| #include "call/adaptation/adaptation_constraint.h" |
| #include "call/adaptation/degradation_preference_provider.h" |
| #include "call/adaptation/video_source_restrictions.h" |
| #include "call/adaptation/video_stream_input_state.h" |
| #include "call/adaptation/video_stream_input_state_provider.h" |
| #include "modules/video_coding/utility/quality_scaler.h" |
| #include "rtc_base/experiments/balanced_degradation_settings.h" |
| #include "rtc_base/system/no_unique_address.h" |
| #include "rtc_base/thread_annotations.h" |
| #include "video/video_stream_encoder_observer.h" |
| |
| namespace webrtc { |
| |
| // The listener is responsible for carrying out the reconfiguration of the video |
| // source such that the VideoSourceRestrictions are fulfilled. |
| class VideoSourceRestrictionsListener { |
| public: |
| virtual ~VideoSourceRestrictionsListener(); |
| |
| // The `restrictions` are filtered by degradation preference but not the |
| // `adaptation_counters`, which are currently only reported for legacy stats |
| // calculation purposes. |
| virtual void OnVideoSourceRestrictionsUpdated( |
| VideoSourceRestrictions restrictions, |
| const VideoAdaptationCounters& adaptation_counters, |
| rtc::scoped_refptr<Resource> reason, |
| const VideoSourceRestrictions& unfiltered_restrictions) = 0; |
| }; |
| |
| class VideoStreamAdapter; |
| |
| extern const int kMinFrameRateFps; |
| |
| VideoSourceRestrictions FilterRestrictionsByDegradationPreference( |
| VideoSourceRestrictions source_restrictions, |
| DegradationPreference degradation_preference); |
| |
| int GetLowerResolutionThan(int pixel_count); |
| int GetHigherResolutionThan(int pixel_count); |
| |
| // Either represents the next VideoSourceRestrictions the VideoStreamAdapter |
| // will take, or provides a Status code indicating the reason for not adapting |
| // if the adaptation is not valid. |
| class Adaptation final { |
| public: |
| enum class Status { |
| // Applying this adaptation will have an effect. All other Status codes |
| // indicate that adaptation is not possible and why. |
| kValid, |
| // Cannot adapt. The minimum or maximum adaptation has already been reached. |
| // There are no more steps to take. |
| kLimitReached, |
| // Cannot adapt. The resolution or frame rate requested by a recent |
| // adaptation has not yet been reflected in the input resolution or frame |
| // rate; adaptation is refused to avoid "double-adapting". |
| kAwaitingPreviousAdaptation, |
| // Not enough input. |
| kInsufficientInput, |
| // Adaptation disabled via degradation preference. |
| kAdaptationDisabled, |
| // Adaptation up was rejected by a VideoAdaptationConstraint. |
| kRejectedByConstraint, |
| }; |
| |
| static const char* StatusToString(Status status); |
| |
| Status status() const; |
| const VideoStreamInputState& input_state() const; |
| const VideoSourceRestrictions& restrictions() const; |
| const VideoAdaptationCounters& counters() const; |
| |
| private: |
| friend class VideoStreamAdapter; |
| |
| // Constructs with a valid adaptation. Status is kValid. |
| Adaptation(int validation_id, |
| VideoSourceRestrictions restrictions, |
| VideoAdaptationCounters counters, |
| VideoStreamInputState input_state); |
| // Constructor when adaptation is not valid. Status MUST NOT be kValid. |
| Adaptation(int validation_id, Status invalid_status); |
| |
| // An Adaptation can become invalidated if the state of VideoStreamAdapter is |
| // modified before the Adaptation is applied. To guard against this, this ID |
| // has to match VideoStreamAdapter::adaptation_validation_id_ when applied. |
| // TODO(https://crbug.com/webrtc/11700): Remove the validation_id_. |
| const int validation_id_; |
| const Status status_; |
| // Input state when adaptation was made. |
| const VideoStreamInputState input_state_; |
| const VideoSourceRestrictions restrictions_; |
| const VideoAdaptationCounters counters_; |
| }; |
| |
| // Owns the VideoSourceRestriction for a single stream and is responsible for |
| // adapting it up or down when told to do so. This class serves the following |
| // purposes: |
| // 1. Keep track of a stream's restrictions. |
| // 2. Provide valid ways to adapt up or down the stream's restrictions. |
| // 3. Modify the stream's restrictions in one of the valid ways. |
| class VideoStreamAdapter { |
| public: |
| VideoStreamAdapter(VideoStreamInputStateProvider* input_state_provider, |
| VideoStreamEncoderObserver* encoder_stats_observer, |
| const FieldTrialsView& field_trials); |
| ~VideoStreamAdapter(); |
| |
| VideoSourceRestrictions source_restrictions() const; |
| const VideoAdaptationCounters& adaptation_counters() const; |
| void ClearRestrictions(); |
| |
| void AddRestrictionsListener( |
| VideoSourceRestrictionsListener* restrictions_listener); |
| void RemoveRestrictionsListener( |
| VideoSourceRestrictionsListener* restrictions_listener); |
| void AddAdaptationConstraint(AdaptationConstraint* adaptation_constraint); |
| void RemoveAdaptationConstraint(AdaptationConstraint* adaptation_constraint); |
| |
| // TODO(hbos): Setting the degradation preference should not clear |
| // restrictions! This is not defined in the spec and is unexpected, there is a |
| // tiny risk that people would discover and rely on this behavior. |
| void SetDegradationPreference(DegradationPreference degradation_preference); |
| |
| // Returns an adaptation that we are guaranteed to be able to apply, or a |
| // status code indicating the reason why we cannot adapt. |
| Adaptation GetAdaptationUp(); |
| Adaptation GetAdaptationDown(); |
| Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, |
| const VideoSourceRestrictions& restrictions); |
| // Tries to adapt the resolution one step. This is used for initial frame |
| // dropping. Does nothing if the degradation preference is not BALANCED or |
| // MAINTAIN_FRAMERATE. In the case of BALANCED, it will try twice to reduce |
| // the resolution. If it fails twice it gives up. |
| Adaptation GetAdaptDownResolution(); |
| |
| // Updates source_restrictions() the Adaptation. |
| void ApplyAdaptation(const Adaptation& adaptation, |
| rtc::scoped_refptr<Resource> resource); |
| |
| struct RestrictionsWithCounters { |
| VideoSourceRestrictions restrictions; |
| VideoAdaptationCounters counters; |
| }; |
| |
| static absl::optional<uint32_t> GetSingleActiveLayerPixels( |
| const VideoCodec& codec); |
| |
| private: |
| void BroadcastVideoRestrictionsUpdate( |
| const VideoStreamInputState& input_state, |
| const rtc::scoped_refptr<Resource>& resource); |
| |
| bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state) |
| const RTC_RUN_ON(&sequence_checker_); |
| |
| using RestrictionsOrState = |
| absl::variant<RestrictionsWithCounters, Adaptation::Status>; |
| RestrictionsOrState GetAdaptationUpStep( |
| const VideoStreamInputState& input_state) const |
| RTC_RUN_ON(&sequence_checker_); |
| RestrictionsOrState GetAdaptationDownStep( |
| const VideoStreamInputState& input_state, |
| const RestrictionsWithCounters& current_restrictions) const |
| RTC_RUN_ON(&sequence_checker_); |
| RestrictionsOrState GetAdaptDownResolutionStepForBalanced( |
| const VideoStreamInputState& input_state) const |
| RTC_RUN_ON(&sequence_checker_); |
| RestrictionsOrState AdaptIfFpsDiffInsufficient( |
| const VideoStreamInputState& input_state, |
| const RestrictionsWithCounters& restrictions) const |
| RTC_RUN_ON(&sequence_checker_); |
| |
| Adaptation GetAdaptationUp(const VideoStreamInputState& input_state) const |
| RTC_RUN_ON(&sequence_checker_); |
| Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const |
| RTC_RUN_ON(&sequence_checker_); |
| |
| static RestrictionsOrState DecreaseResolution( |
| const VideoStreamInputState& input_state, |
| const RestrictionsWithCounters& current_restrictions); |
| static RestrictionsOrState IncreaseResolution( |
| const VideoStreamInputState& input_state, |
| const RestrictionsWithCounters& current_restrictions); |
| // Framerate methods are member functions because they need internal state |
| // if the degradation preference is BALANCED. |
| RestrictionsOrState DecreaseFramerate( |
| const VideoStreamInputState& input_state, |
| const RestrictionsWithCounters& current_restrictions) const |
| RTC_RUN_ON(&sequence_checker_); |
| RestrictionsOrState IncreaseFramerate( |
| const VideoStreamInputState& input_state, |
| const RestrictionsWithCounters& current_restrictions) const |
| RTC_RUN_ON(&sequence_checker_); |
| |
| struct RestrictionsOrStateVisitor; |
| Adaptation RestrictionsOrStateToAdaptation( |
| RestrictionsOrState step_or_state, |
| const VideoStreamInputState& input_state) const |
| RTC_RUN_ON(&sequence_checker_); |
| |
| RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_ |
| RTC_GUARDED_BY(&sequence_checker_); |
| // Gets the input state which is the basis of all adaptations. |
| // Thread safe. |
| VideoStreamInputStateProvider* input_state_provider_; |
| // Used to signal when min pixel limit has been reached. |
| VideoStreamEncoderObserver* const encoder_stats_observer_; |
| // Decides the next adaptation target in DegradationPreference::BALANCED. |
| const BalancedDegradationSettings balanced_settings_; |
| // To guard against applying adaptations that have become invalidated, an |
| // Adaptation that is applied has to have a matching validation ID. |
| int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_); |
| // When deciding the next target up or down, different strategies are used |
| // depending on the DegradationPreference. |
| // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference |
| DegradationPreference degradation_preference_ |
| RTC_GUARDED_BY(&sequence_checker_); |
| // Used to avoid adapting twice. Stores the resolution at the time of the last |
| // adaptation. |
| // TODO(hbos): Can we implement a more general "cooldown" mechanism of |
| // resources intead? If we already have adapted it seems like we should wait |
| // a while before adapting again, so that we are not acting on usage |
| // measurements that are made obsolete/unreliable by an "ongoing" adaptation. |
| struct AwaitingFrameSizeChange { |
| AwaitingFrameSizeChange(bool pixels_increased, int frame_size); |
| const bool pixels_increased; |
| const int frame_size_pixels; |
| }; |
| absl::optional<AwaitingFrameSizeChange> awaiting_frame_size_change_ |
| RTC_GUARDED_BY(&sequence_checker_); |
| // The previous restrictions value. Starts as unrestricted. |
| VideoSourceRestrictions last_video_source_restrictions_ |
| RTC_GUARDED_BY(&sequence_checker_); |
| VideoSourceRestrictions last_filtered_restrictions_ |
| RTC_GUARDED_BY(&sequence_checker_); |
| |
| std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_ |
| RTC_GUARDED_BY(&sequence_checker_); |
| std::vector<AdaptationConstraint*> adaptation_constraints_ |
| RTC_GUARDED_BY(&sequence_checker_); |
| |
| RestrictionsWithCounters current_restrictions_ |
| RTC_GUARDED_BY(&sequence_checker_); |
| }; |
| |
| } // namespace webrtc |
| |
| #endif // CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ |