[Adaptation] Adapter: Inform the module why there is no next target.
This introduces AdaptationTargetOrReason and gets rid of
VideoStreamAdapter's dependency on the VideoStreamEncoderObserver.
AdaptationTargetOrReason provides information about why an adaptation
target could not be returned from GetAdaptUpTarget() and
GetAdaptDownTarget() with the enum CannotAdaptReason and the boolean
min_pixel_limit_reached.
While the enum value is not used by the caller in this CL, it makes
explicit reasons the adapter is allowed to reject a target. TODOs are
added documenting how we want to get rid of kAwaitingPreviousAdaptation
for multi-stream use cases and how kIsBitrateConstrained can be
rephrased as a resource problem in the future.
min_pixel_limit_reached() allows us to move the responsibility of stats
reporting to the module. A TODO documents how this could be replaced by
kLimitReached or similar logic in the future.
Bug: webrtc:11393
Change-Id: Iffdd8ddb01641937741fac353174ea14168477ce
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169928
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30789}
diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn
index f8bb6d8..f4e9792 100644
--- a/video/adaptation/BUILD.gn
+++ b/video/adaptation/BUILD.gn
@@ -50,6 +50,7 @@
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/base:core_headers",
"//third_party/abseil-cpp/absl/types:optional",
+ "//third_party/abseil-cpp/absl/types:variant",
]
}
diff --git a/video/adaptation/resource_adaptation_processor.cc b/video/adaptation/resource_adaptation_processor.cc
index 7958be4..eccf0f2 100644
--- a/video/adaptation/resource_adaptation_processor.cc
+++ b/video/adaptation/resource_adaptation_processor.cc
@@ -461,15 +461,16 @@
const int input_pixels = LastInputFrameSizeOrDefault();
const int input_fps = encoder_stats_observer_->GetInputFrameRate();
// Should we adapt, if so to what target?
- absl::optional<VideoStreamAdapter::AdaptationTarget> target =
+ VideoStreamAdapter::AdaptationTargetOrReason target_or_reason =
stream_adapter_->GetAdaptUpTarget(encoder_settings_,
encoder_target_bitrate_bps_, input_mode,
input_pixels, input_fps, reason);
- if (!target.has_value())
+ if (!target_or_reason.has_target())
return;
// Apply target.
- stream_adapter_->ApplyAdaptationTarget(target.value(), encoder_settings_,
- input_mode, input_pixels, input_fps);
+ stream_adapter_->ApplyAdaptationTarget(target_or_reason.target(),
+ encoder_settings_, input_mode,
+ input_pixels, input_fps);
// Update VideoSourceRestrictions based on adaptation. This also informs the
// |adaptation_listener_|.
MaybeUpdateVideoSourceRestrictions();
@@ -487,15 +488,17 @@
const int input_pixels = LastInputFrameSizeOrDefault();
const int input_fps = encoder_stats_observer_->GetInputFrameRate();
// Should we adapt, if so to what target?
- absl::optional<VideoStreamAdapter::AdaptationTarget> target =
+ VideoStreamAdapter::AdaptationTargetOrReason target_or_reason =
stream_adapter_->GetAdaptDownTarget(encoder_settings_, input_mode,
- input_pixels, input_fps,
- encoder_stats_observer_);
- if (!target.has_value())
+ input_pixels, input_fps);
+ if (target_or_reason.min_pixel_limit_reached())
+ encoder_stats_observer_->OnMinPixelLimitReached();
+ if (!target_or_reason.has_target())
return ResourceListenerResponse::kNothing;
// Apply target.
ResourceListenerResponse response = stream_adapter_->ApplyAdaptationTarget(
- target.value(), encoder_settings_, input_mode, input_pixels, input_fps);
+ target_or_reason.target(), encoder_settings_, input_mode, input_pixels,
+ input_fps);
// Update VideoSourceRestrictions based on adaptation. This also informs the
// |adaptation_listener_|.
MaybeUpdateVideoSourceRestrictions();
diff --git a/video/adaptation/video_stream_adapter.cc b/video/adaptation/video_stream_adapter.cc
index 078410a..d0c0ebe 100644
--- a/video/adaptation/video_stream_adapter.cc
+++ b/video/adaptation/video_stream_adapter.cc
@@ -12,8 +12,10 @@
#include <algorithm>
#include <limits>
+#include <utility>
#include "absl/types/optional.h"
+#include "absl/types/variant.h"
#include "api/video_codecs/video_encoder.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/logging.h"
@@ -98,6 +100,49 @@
int value)
: action(action), value(value) {}
+VideoStreamAdapter::AdaptationTargetOrReason::AdaptationTargetOrReason(
+ AdaptationTarget target,
+ bool min_pixel_limit_reached)
+ : target_or_reason_(target),
+ min_pixel_limit_reached_(min_pixel_limit_reached) {}
+
+VideoStreamAdapter::AdaptationTargetOrReason::AdaptationTargetOrReason(
+ CannotAdaptReason reason,
+ bool min_pixel_limit_reached)
+ : target_or_reason_(reason),
+ min_pixel_limit_reached_(min_pixel_limit_reached) {}
+
+// implicit
+VideoStreamAdapter::AdaptationTargetOrReason::AdaptationTargetOrReason(
+ AdaptationTarget target)
+ : target_or_reason_(target), min_pixel_limit_reached_(false) {}
+
+// implicit
+VideoStreamAdapter::AdaptationTargetOrReason::AdaptationTargetOrReason(
+ CannotAdaptReason reason)
+ : target_or_reason_(reason), min_pixel_limit_reached_(false) {}
+
+bool VideoStreamAdapter::AdaptationTargetOrReason::has_target() const {
+ return absl::holds_alternative<AdaptationTarget>(target_or_reason_);
+}
+
+const VideoStreamAdapter::AdaptationTarget&
+VideoStreamAdapter::AdaptationTargetOrReason::target() const {
+ RTC_DCHECK(has_target());
+ return absl::get<AdaptationTarget>(target_or_reason_);
+}
+
+VideoStreamAdapter::CannotAdaptReason
+VideoStreamAdapter::AdaptationTargetOrReason::reason() const {
+ RTC_DCHECK(!has_target());
+ return absl::get<CannotAdaptReason>(target_or_reason_);
+}
+
+bool VideoStreamAdapter::AdaptationTargetOrReason::min_pixel_limit_reached()
+ const {
+ return min_pixel_limit_reached_;
+}
+
// VideoSourceRestrictor is responsible for keeping track of current
// VideoSourceRestrictions.
class VideoStreamAdapter::VideoSourceRestrictor {
@@ -270,7 +315,7 @@
: SetDegradationPreferenceResult::kRestrictionsNotCleared;
}
-absl::optional<VideoStreamAdapter::AdaptationTarget>
+VideoStreamAdapter::AdaptationTargetOrReason
VideoStreamAdapter::GetAdaptUpTarget(
const absl::optional<EncoderSettings>& encoder_settings,
absl::optional<uint32_t> encoder_target_bitrate_bps,
@@ -278,32 +323,31 @@
int input_pixels,
int input_fps,
AdaptationObserverInterface::AdaptReason reason) const {
- // Preconditions for being able to adapt up:
- if (input_mode == VideoInputMode::kNoVideo)
- return absl::nullopt;
- // 1. We shouldn't adapt up if we're currently waiting for a previous upgrade
- // to have an effect.
- // TODO(hbos): What about in the case of other degradation preferences?
+ // Don't adapt if we don't have sufficient input.
+ if (input_mode == VideoInputMode::kNoVideo) {
+ return CannotAdaptReason::kInsufficientInput;
+ }
+ // Don't adapt if we're awaiting a previous adaptation to have an effect.
bool last_adaptation_was_up =
last_adaptation_request_ &&
last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
if (last_adaptation_was_up &&
degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
input_pixels <= last_adaptation_request_->input_pixel_count_) {
- return absl::nullopt;
+ return CannotAdaptReason::kAwaitingPreviousAdaptation;
}
- // 2. We shouldn't adapt up if BalancedSettings doesn't allow it, which is
- // only applicable if reason is kQuality and preference is BALANCED.
+ // Don't adapt if BalancedDegradationSettings applies and determines this will
+ // exceed bitrate constraints.
if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
EffectiveDegradationPreference(input_mode) ==
DegradationPreference::BALANCED &&
!balanced_settings_.CanAdaptUp(
GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels,
encoder_target_bitrate_bps.value_or(0))) {
- return absl::nullopt;
+ return CannotAdaptReason::kIsBitrateConstrained;
}
- // Attempt to find an allowed adaptation target.
+ // Maybe propose targets based on degradation preference.
switch (EffectiveDegradationPreference(input_mode)) {
case DegradationPreference::BALANCED: {
// Attempt to increase target frame rate.
@@ -319,7 +363,7 @@
!balanced_settings_.CanAdaptUpResolution(
GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels,
encoder_target_bitrate_bps.value_or(0))) {
- return absl::nullopt;
+ return CannotAdaptReason::kIsBitrateConstrained;
}
// Scale up resolution.
ABSL_FALLTHROUGH_INTENDED;
@@ -330,7 +374,7 @@
if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
!CanAdaptUpResolution(encoder_settings, encoder_target_bitrate_bps,
input_pixels)) {
- return absl::nullopt;
+ return CannotAdaptReason::kIsBitrateConstrained;
}
// Attempt to increase pixel count.
int target_pixels = input_pixels;
@@ -340,8 +384,9 @@
target_pixels = std::numeric_limits<int>::max();
}
target_pixels = GetHigherResolutionThan(target_pixels);
- if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels))
- return absl::nullopt;
+ if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels)) {
+ return CannotAdaptReason::kLimitReached;
+ }
return AdaptationTarget(AdaptationAction::kIncreaseResolution,
target_pixels);
}
@@ -353,57 +398,50 @@
target_fps = std::numeric_limits<int>::max();
}
target_fps = GetHigherFrameRateThan(target_fps);
- if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps))
- return absl::nullopt;
+ if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps)) {
+ return CannotAdaptReason::kLimitReached;
+ }
return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
}
case DegradationPreference::DISABLED:
- return absl::nullopt;
+ return CannotAdaptReason::kAdaptationDisabled;
}
}
-absl::optional<VideoStreamAdapter::AdaptationTarget>
+VideoStreamAdapter::AdaptationTargetOrReason
VideoStreamAdapter::GetAdaptDownTarget(
const absl::optional<EncoderSettings>& encoder_settings,
VideoInputMode input_mode,
int input_pixels,
- int input_fps,
- VideoStreamEncoderObserver* encoder_stats_observer) const {
+ int input_fps) const {
const int min_pixels_per_frame = MinPixelsPerFrame(encoder_settings);
- // Preconditions for being able to adapt down:
- if (input_mode == VideoInputMode::kNoVideo)
- return absl::nullopt;
- // 1. We are not disabled.
- // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
- // causes scaling due to bandwidth constraints (QualityScalerResource) to be
- // ignored, not just CPU signals. This is not a use case we want to support
- // long-term; remove this enum value.
- if (degradation_preference_ == DegradationPreference::DISABLED)
- return absl::nullopt;
+ // Don't adapt if we don't have sufficient input or adaptation is disabled.
+ if (input_mode == VideoInputMode::kNoVideo) {
+ return CannotAdaptReason::kInsufficientInput;
+ }
+ if (degradation_preference_ == DegradationPreference::DISABLED) {
+ return CannotAdaptReason::kAdaptationDisabled;
+ }
bool last_adaptation_was_down =
last_adaptation_request_ &&
last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
- // 2. We shouldn't adapt down if our frame rate is below the minimum or if its
- // currently unknown.
if (EffectiveDegradationPreference(input_mode) ==
DegradationPreference::MAINTAIN_RESOLUTION) {
// TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
// - delete it.
if (input_fps <= 0 ||
(last_adaptation_was_down && input_fps < kMinFramerateFps)) {
- return absl::nullopt;
+ return CannotAdaptReason::kInsufficientInput;
}
}
- // 3. We shouldn't adapt down if we're currently waiting for a previous
- // downgrade to have an effect.
- // TODO(hbos): What about in the case of other degradation preferences?
+ // Don't adapt if we're awaiting a previous adaptation to have an effect.
if (last_adaptation_was_down &&
degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
input_pixels >= last_adaptation_request_->input_pixel_count_) {
- return absl::nullopt;
+ return CannotAdaptReason::kAwaitingPreviousAdaptation;
}
- // Attempt to find an allowed adaptation target.
+ // Maybe propose targets based on degradation preference.
switch (EffectiveDegradationPreference(input_mode)) {
case DegradationPreference::BALANCED: {
// Try scale down framerate, if lower.
@@ -419,27 +457,27 @@
case DegradationPreference::MAINTAIN_FRAMERATE: {
// Scale down resolution.
int target_pixels = GetLowerResolutionThan(input_pixels);
- // TODO(https://crbug.com/webrtc/11393): Move this logic to
- // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
- // target is available should not have side-effects.
- if (target_pixels < min_pixels_per_frame)
- encoder_stats_observer->OnMinPixelLimitReached();
+ bool min_pixel_limit_reached = target_pixels < min_pixels_per_frame;
if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels,
min_pixels_per_frame)) {
- return absl::nullopt;
+ return AdaptationTargetOrReason(CannotAdaptReason::kLimitReached,
+ min_pixel_limit_reached);
}
- return AdaptationTarget(AdaptationAction::kDecreaseResolution,
- target_pixels);
+ return AdaptationTargetOrReason(
+ AdaptationTarget(AdaptationAction::kDecreaseResolution,
+ target_pixels),
+ min_pixel_limit_reached);
}
case DegradationPreference::MAINTAIN_RESOLUTION: {
int target_fps = GetLowerFrameRateThan(input_fps);
- if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps))
- return absl::nullopt;
+ if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) {
+ return CannotAdaptReason::kLimitReached;
+ }
return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
}
case DegradationPreference::DISABLED:
RTC_NOTREACHED();
- return absl::nullopt;
+ return CannotAdaptReason::kAdaptationDisabled;
}
}
diff --git a/video/adaptation/video_stream_adapter.h b/video/adaptation/video_stream_adapter.h
index d4d9ff2..16b1b44 100644
--- a/video/adaptation/video_stream_adapter.h
+++ b/video/adaptation/video_stream_adapter.h
@@ -14,8 +14,8 @@
#include <memory>
#include "absl/types/optional.h"
+#include "absl/types/variant.h"
#include "api/rtp_parameters.h"
-#include "api/video/video_stream_encoder_observer.h"
#include "call/adaptation/encoder_settings.h"
#include "call/adaptation/resource.h"
#include "call/adaptation/video_source_restrictions.h"
@@ -69,6 +69,78 @@
friend class absl::optional<AdaptationTarget>;
};
+ // Reasons for not being able to get an AdaptationTarget that can be applied.
+ enum class CannotAdaptReason {
+ // DegradationPreference is DISABLED.
+ // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
+ // causes all adaptation to be ignored, even QP-scaling.
+ kAdaptationDisabled,
+ // Adaptation is refused because we don't have video, the input frame rate
+ // is not known yet or is less than the minimum allowed (below the limit).
+ kInsufficientInput,
+ // The minimum or maximum adaptation has already been reached. There are no
+ // more steps to take.
+ kLimitReached,
+ // The resolution or frame rate requested by a recent adaptation has not yet
+ // been reflected in the input resolution or frame rate; adaptation is
+ // refused to avoid "double-adapting".
+ // TODO(hbos): Can this be rephrased as a resource usage measurement
+ // cooldown mechanism? In a multi-stream setup, we need to wait before
+ // adapting again across streams. The best way to achieve this is probably
+ // to not act on racy resource usage measurements, regardless of individual
+ // adapters. When this logic is moved or replaced then remove this enum
+ // value.
+ kAwaitingPreviousAdaptation,
+ // The adaptation that would have been proposed by the adapter violates
+ // bitrate constraints and is therefore rejected.
+ // TODO(hbos): This is a version of being resource limited, except in order
+ // to know if we are constrained we need to have a proposed adaptation in
+ // mind, thus the resource alone cannot determine this in isolation.
+ // Proposal: ask resources for permission to apply a proposed adaptation.
+ // This allows rejecting a given resolution or frame rate based on bitrate
+ // limits without coupling it with the adapter's proposal logic. When this
+ // is done, remove this enum value.
+ kIsBitrateConstrained,
+ };
+
+ // Describes the next adaptation target that can be applied, or a reason
+ // explaining why there is no next adaptation step to take.
+ // TODO(hbos): Make "AdaptationTarget" a private implementation detail and
+ // expose the resulting VideoSourceRestrictions as the publically accessible
+ // "target" instead.
+ class AdaptationTargetOrReason {
+ public:
+ AdaptationTargetOrReason(AdaptationTarget target,
+ bool min_pixel_limit_reached);
+ AdaptationTargetOrReason(CannotAdaptReason reason,
+ bool min_pixel_limit_reached);
+ // Not explicit - we want to use AdaptationTarget and CannotAdaptReason as
+ // return values.
+ AdaptationTargetOrReason(AdaptationTarget target); // NOLINT
+ AdaptationTargetOrReason(CannotAdaptReason reason); // NOLINT
+
+ bool has_target() const;
+ const AdaptationTarget& target() const;
+ CannotAdaptReason reason() const;
+ // This is true if the next step down would have exceeded the minimum
+ // resolution limit. Used for stats reporting. This is similar to
+ // kLimitReached but only applies to resolution adaptations. It is also
+ // currently implemented as "the next step would have exceeded", which is
+ // subtly diffrent than "we are currently reaching the limit" - we could
+ // stay above the limit forever, not taking any steps because the steps
+ // would have been too big. (This is unlike how we adapt frame rate, where
+ // we adapt to kMinFramerateFps before reporting kLimitReached.)
+ // TODO(hbos): Adapt to the limit and indicate if the limit was reached
+ // independently of degradation preference. If stats reporting wants to
+ // filter this out by degradation preference it can take on that
+ // responsibility; the adapter should not inherit this detail.
+ bool min_pixel_limit_reached() const;
+
+ private:
+ const absl::variant<AdaptationTarget, CannotAdaptReason> target_or_reason_;
+ const bool min_pixel_limit_reached_;
+ };
+
VideoStreamAdapter();
~VideoStreamAdapter();
@@ -77,7 +149,7 @@
// TODO(hbos): Can we get rid of any external dependencies on
// BalancedDegradationPreference? How the adaptor generates possible next
// steps for adaptation should be an implementation detail. Can the relevant
- // information be inferred from GetAdaptUpTarget()/GetAdaptDownTarget()?
+ // information be inferred from AdaptationTargetOrReason?
const BalancedDegradationSettings& balanced_settings() const;
void ClearRestrictions();
@@ -87,24 +159,20 @@
SetDegradationPreferenceResult SetDegradationPreference(
DegradationPreference degradation_preference);
- // Returns a target that we are guaranteed to be able to adapt to, or null if
- // adaptation is not desired or not possible.
- absl::optional<AdaptationTarget> GetAdaptUpTarget(
+ // Returns a target that we are guaranteed to be able to adapt to, or the
+ // reason why there is no such target.
+ AdaptationTargetOrReason GetAdaptUpTarget(
const absl::optional<EncoderSettings>& encoder_settings,
absl::optional<uint32_t> encoder_target_bitrate_bps,
VideoInputMode input_mode,
int input_pixels,
int input_fps,
AdaptationObserverInterface::AdaptReason reason) const;
- // TODO(https://crbug.com/webrtc/11393): Remove the dependency on
- // |encoder_stats_observer| - simply checking which adaptation target is
- // available should not have side-effects.
- absl::optional<AdaptationTarget> GetAdaptDownTarget(
+ AdaptationTargetOrReason GetAdaptDownTarget(
const absl::optional<EncoderSettings>& encoder_settings,
VideoInputMode input_mode,
int input_pixels,
- int input_fps,
- VideoStreamEncoderObserver* encoder_stats_observer) const;
+ int input_fps) const;
// Applies the |target| to |source_restrictor_|.
// TODO(hbos): Delete ResourceListenerResponse!
ResourceListenerResponse ApplyAdaptationTarget(
