[Overuse] Setting encoder configurations through the interface.
This squashes together several input signals that were spread out
through several calls into a single method and calling place:
SetEncoderSettings(), invoked from ReconfigureEncoder(). This is added
to the abstract interface.
This makes the following methods obsolete which are removed:
- SetEncoder(): The VideoEncoder was only used for GetEncoderInfo();
the VideoEncoder::EncoderInfo is now part of the EncoderSettings.
- SetEncoderConfig(): The VideoEncoderConfig is part of
EncoderSettings. The config is used for its codec_type and
content_type enums.
- SetCodecMaxFrameRate(): The max frame rate was the same as
VideoCodec::maxFramerate. VideoCodec is now part of EncoderSettings.
There may be some overlap in information between EncoderConfig and
VideoCodec, but that is outside the scope of this CL, which only makes
sure to bundle encoder settings-like information into one input signal.
Bug: webrtc:11222
Change-Id: I67c49c49c0a859cb7d5051939a461593c695a789
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/166602
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30332}
diff --git a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
index a26c0c3..574bc6f 100644
--- a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
+++ b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
@@ -48,7 +48,6 @@
const int kNumCores = 2;
const uint32_t kFramerate = 30;
const size_t kMaxPayloadSize = 800;
-const int kDefaultMinPixelsPerFrame = 320 * 180;
const int kLowThreshold = 10;
const int kHighThreshold = 20;
diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h
index 061bdb8..34502c8 100644
--- a/api/video_codecs/video_encoder.h
+++ b/api/video_codecs/video_encoder.h
@@ -34,6 +34,8 @@
// TODO(pbos): Expose these through a public (root) header or change these APIs.
struct CodecSpecificInfo;
+constexpr int kDefaultMinPixelsPerFrame = 320 * 180;
+
class EncodedImageCallback {
public:
virtual ~EncodedImageCallback() {}
@@ -115,7 +117,7 @@
// TODO(kthelgason): Lower this limit when better testing
// on MediaCodec and fallback implementations are in place.
// See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206
- int min_pixels_per_frame = 320 * 180;
+ int min_pixels_per_frame = kDefaultMinPixelsPerFrame;
private:
// Private constructor; to get an object without thresholds, use
diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn
index ef1c6e6..99b3f16 100644
--- a/call/adaptation/BUILD.gn
+++ b/call/adaptation/BUILD.gn
@@ -25,6 +25,7 @@
]
deps = [
"../../api:rtp_parameters",
+ "../../api/video_codecs:video_codecs_api",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"//third_party/abseil-cpp/absl/types:optional",
diff --git a/call/adaptation/resource_adaptation_module_interface.cc b/call/adaptation/resource_adaptation_module_interface.cc
index e89d1ef..63cfb72 100644
--- a/call/adaptation/resource_adaptation_module_interface.cc
+++ b/call/adaptation/resource_adaptation_module_interface.cc
@@ -10,8 +10,29 @@
#include "call/adaptation/resource_adaptation_module_interface.h"
+#include <utility>
+
namespace webrtc {
+EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
+ VideoEncoderConfig encoder_config,
+ VideoCodec video_codec)
+ : encoder_info_(std::move(encoder_info)),
+ encoder_config_(std::move(encoder_config)),
+ video_codec_(std::move(video_codec)) {}
+
+const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const {
+ return encoder_info_;
+}
+
+const VideoEncoderConfig& EncoderSettings::encoder_config() const {
+ return encoder_config_;
+}
+
+const VideoCodec& EncoderSettings::video_codec() const {
+ return video_codec_;
+}
+
ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {}
ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {}
diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h
index d71ffe8..bc64b8e 100644
--- a/call/adaptation/resource_adaptation_module_interface.h
+++ b/call/adaptation/resource_adaptation_module_interface.h
@@ -12,10 +12,32 @@
#define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_
#include "api/rtp_parameters.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/video_encoder_config.h"
#include "call/adaptation/video_source_restrictions.h"
namespace webrtc {
+// Information about an encoder available when reconfiguring the encoder.
+class EncoderSettings {
+ public:
+ EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
+ VideoEncoderConfig encoder_config,
+ VideoCodec video_codec);
+
+ // Encoder capabilities, implementation info, etc.
+ const VideoEncoder::EncoderInfo& encoder_info() const;
+ // Configuration parameters, ultimately coming from the API and negotiation.
+ const VideoEncoderConfig& encoder_config() const;
+ // Lower level config, heavily based on the VideoEncoderConfig.
+ const VideoCodec& video_codec() const;
+
+ private:
+ VideoEncoder::EncoderInfo encoder_info_;
+ VideoEncoderConfig encoder_config_;
+ VideoCodec video_codec_;
+};
+
// The listener is responsible for carrying out the reconfiguration of the video
// source such that the VideoSourceRestrictions are fulfilled.
class ResourceAdaptationModuleListener {
@@ -61,6 +83,7 @@
virtual void SetHasInputVideo(bool has_input_video) = 0;
virtual void SetDegradationPreference(
DegradationPreference degradation_preference) = 0;
+ virtual void SetEncoderSettings(EncoderSettings encoder_settings) = 0;
// Removes all restrictions; the module will need to adapt all over again.
// TODO(hbos): It's not clear why anybody should be able to tell the module to
// reset like this; can we get rid of this method?
diff --git a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
index 4c80c05..a597dc4 100644
--- a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
@@ -50,7 +50,6 @@
constexpr int64_t kInitialTimestampMs = 789;
constexpr int kNumCores = 1;
constexpr size_t kMaxPayloadSize = 1440;
-constexpr int kDefaultMinPixelsPerFrame = 320 * 180;
constexpr int kWidth = 172;
constexpr int kHeight = 144;
constexpr float kFramerateFps = 30;
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
index 2ca3435..bd27eda 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.cc
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -356,12 +356,10 @@
source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
overuse_detector_(std::move(overuse_detector)),
overuse_detector_is_started_(false),
- codec_max_frame_rate_(absl::nullopt),
target_frame_rate_(absl::nullopt),
encoder_start_bitrate_bps_(0),
is_quality_scaler_enabled_(false),
- encoder_config_(),
- encoder_(nullptr),
+ encoder_settings_(absl::nullopt),
encoder_stats_observer_(encoder_stats_observer) {
RTC_DCHECK(adaptation_listener_);
RTC_DCHECK(video_stream_encoder_);
@@ -372,14 +370,9 @@
OveruseFrameDetectorResourceAdaptationModule::
~OveruseFrameDetectorResourceAdaptationModule() {}
-void OveruseFrameDetectorResourceAdaptationModule::SetEncoder(
- VideoEncoder* encoder) {
- encoder_ = encoder;
-}
-
void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation(
ResourceAdaptationModuleListener* adaptation_listener) {
- RTC_DCHECK(encoder_);
+ RTC_DCHECK(encoder_settings_.has_value());
RTC_DCHECK(!overuse_detector_is_started_);
// TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside
// the interval between StartCheckForOveruse() and StopCheckForOveruse(),
@@ -427,6 +420,12 @@
MaybeUpdateVideoSourceRestrictions();
}
+void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings(
+ EncoderSettings encoder_settings) {
+ encoder_settings_ = std::move(encoder_settings);
+ MaybeUpdateTargetFrameRate();
+}
+
void OveruseFrameDetectorResourceAdaptationModule::
ResetVideoSourceRestrictions() {
last_adaptation_request_.reset();
@@ -473,19 +472,6 @@
last_frame_pixel_count_ = last_frame_pixel_count;
}
-void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig(
- VideoEncoderConfig encoder_config) {
- encoder_config_ = std::move(encoder_config);
-}
-
-void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFrameRate(
- absl::optional<double> codec_max_frame_rate) {
- RTC_DCHECK(!codec_max_frame_rate.has_value() ||
- codec_max_frame_rate.value() > 0.0);
- codec_max_frame_rate_ = codec_max_frame_rate;
- MaybeUpdateTargetFrameRate();
-}
-
void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps(
uint32_t encoder_start_bitrate_bps) {
encoder_start_bitrate_bps_ = encoder_start_bitrate_bps;
@@ -528,13 +514,13 @@
case DegradationPreference::BALANCED: {
// Check if quality should be increased based on bitrate.
if (reason == kQuality &&
- !balanced_settings_.CanAdaptUp(encoder_config_.codec_type,
+ !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(),
*last_frame_pixel_count_,
encoder_start_bitrate_bps_)) {
return;
}
// Try scale up framerate, if higher.
- int fps = balanced_settings_.MaxFps(encoder_config_.codec_type,
+ int fps = balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(),
*last_frame_pixel_count_);
if (source_restrictor_->IncreaseFramerate(fps)) {
GetAdaptCounter().DecrementFramerate(reason, fps);
@@ -550,7 +536,7 @@
// Check if resolution should be increased based on bitrate.
if (reason == kQuality &&
!balanced_settings_.CanAdaptUpResolution(
- encoder_config_.codec_type, *last_frame_pixel_count_,
+ GetVideoCodecTypeOrGeneric(), *last_frame_pixel_count_,
encoder_start_bitrate_bps_)) {
return;
}
@@ -654,7 +640,7 @@
switch (EffectiveDegradataionPreference()) {
case DegradationPreference::BALANCED: {
// Try scale down framerate, if lower.
- int fps = balanced_settings_.MinFps(encoder_config_.codec_type,
+ int fps = balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(),
*last_frame_pixel_count_);
if (source_restrictor_->RestrictFramerate(fps)) {
GetAdaptCounter().IncrementFramerate(reason);
@@ -677,7 +663,10 @@
bool min_pixels_reached = false;
if (!source_restrictor_->RequestResolutionLowerThan(
adaptation_request.input_pixel_count_,
- encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame,
+ encoder_settings_.has_value()
+ ? encoder_settings_->encoder_info()
+ .scaling_settings.min_pixels_per_frame
+ : kDefaultMinPixelsPerFrame,
&min_pixels_reached)) {
if (min_pixels_reached)
encoder_stats_observer_->OnMinPixelLimitReached();
@@ -712,6 +701,14 @@
return did_adapt;
}
+VideoCodecType
+OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric()
+ const {
+ return encoder_settings_.has_value()
+ ? encoder_settings_->encoder_config().codec_type
+ : kVideoCodecGeneric;
+}
+
void OveruseFrameDetectorResourceAdaptationModule::
MaybeUpdateVideoSourceRestrictions() {
VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
@@ -726,6 +723,11 @@
void OveruseFrameDetectorResourceAdaptationModule::
MaybeUpdateTargetFrameRate() {
+ absl::optional<double> codec_max_frame_rate =
+ encoder_settings_.has_value()
+ ? absl::optional<double>(
+ encoder_settings_->video_codec().maxFramerate)
+ : absl::nullopt;
// The current target framerate is the maximum frame rate as specified by
// the current codec configuration or any limit imposed by the adaptation
// module. This is used to make sure overuse detection doesn't needlessly
@@ -735,9 +737,9 @@
degradation_preference_)
.max_frame_rate();
if (!target_frame_rate.has_value() ||
- (codec_max_frame_rate_.has_value() &&
- codec_max_frame_rate_.value() < target_frame_rate.value())) {
- target_frame_rate = codec_max_frame_rate_;
+ (codec_max_frame_rate.has_value() &&
+ codec_max_frame_rate.value() < target_frame_rate.value())) {
+ target_frame_rate = codec_max_frame_rate;
}
if (target_frame_rate != target_frame_rate_) {
target_frame_rate_ = target_frame_rate;
@@ -799,7 +801,8 @@
// Resolution is capped for fullscreen animated content.
// Adapatation is done only via framerate downgrade.
// Thus effective degradation preference is MAINTAIN_RESOLUTION.
- return (encoder_config_.content_type ==
+ return (encoder_settings_.has_value() &&
+ encoder_settings_->encoder_config().content_type ==
VideoEncoderConfig::ContentType::kScreen &&
degradation_preference_ == DegradationPreference::BALANCED)
? DegradationPreference::MAINTAIN_RESOLUTION
@@ -819,7 +822,7 @@
absl::optional<VideoEncoder::QpThresholds>
OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const {
RTC_DCHECK(last_frame_pixel_count_.has_value());
- return balanced_settings_.GetQpThresholds(encoder_config_.codec_type,
+ return balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
last_frame_pixel_count_.value());
}
@@ -827,9 +830,11 @@
int pixels,
uint32_t bitrate_bps) const {
absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
- GetEncoderBitrateLimits(
- encoder_->GetEncoderInfo(),
- source_restrictor_->GetHigherResolutionThan(pixels));
+ encoder_settings_.has_value()
+ ? GetEncoderBitrateLimits(
+ encoder_settings_->encoder_info(),
+ source_restrictor_->GetHigherResolutionThan(pixels))
+ : absl::nullopt;
if (!bitrate_limits.has_value() || bitrate_bps == 0) {
return true; // No limit configured or bitrate provided.
}
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
index 798d386..efc2ec8 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.h
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -22,6 +22,7 @@
#include "api/video/video_frame.h"
#include "api/video/video_source_interface.h"
#include "api/video/video_stream_encoder_observer.h"
+#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_config.h"
#include "call/adaptation/resource_adaptation_module_interface.h"
@@ -60,12 +61,6 @@
ResourceAdaptationModuleListener* adaptation_listener);
~OveruseFrameDetectorResourceAdaptationModule() override;
- // Sets the encoder to reconfigure based on overuse.
- // TODO(hbos): Don't reconfigure the encoder directly. Instead, define the
- // output of a resource adaptation module as a struct and let the
- // VideoStreamEncoder handle the interaction with the actual encoder.
- void SetEncoder(VideoEncoder* encoder);
-
DegradationPreference degradation_preference() const {
return degradation_preference_;
}
@@ -77,6 +72,7 @@
void SetHasInputVideo(bool has_input_video) override;
void SetDegradationPreference(
DegradationPreference degradation_preference) override;
+ void SetEncoderSettings(EncoderSettings encoder_settings) override;
void ResetVideoSourceRestrictions() override;
// Input to the OveruseFrameDetector, which are required for this module to
@@ -96,8 +92,6 @@
// resource adaptation module. Unify code paths where possible. Do we really
// need this many public methods?
void SetLastFramePixelCount(absl::optional<int> last_frame_pixel_count);
- void SetEncoderConfig(VideoEncoderConfig encoder_config);
- void SetCodecMaxFrameRate(absl::optional<double> codec_max_frame_rate);
void SetEncoderStartBitrateBps(uint32_t encoder_start_bitrate_bps);
// Inform the detector whether or not the quality scaler is enabled. This
// helps GetActiveCounts() return absl::nullopt when appropriate.
@@ -182,6 +176,8 @@
enum class Mode { kAdaptUp, kAdaptDown } mode_;
};
+ VideoCodecType GetVideoCodecTypeOrGeneric() const;
+
// Makes |video_source_restrictions_| up-to-date and informs the
// |adaptation_listener_| if restrictions are changed, allowing the listener
// to reconfigure the source accordingly.
@@ -218,12 +214,10 @@
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
const std::unique_ptr<OveruseFrameDetector> overuse_detector_;
bool overuse_detector_is_started_;
- absl::optional<double> codec_max_frame_rate_;
absl::optional<double> target_frame_rate_;
uint32_t encoder_start_bitrate_bps_;
bool is_quality_scaler_enabled_;
- VideoEncoderConfig encoder_config_;
- VideoEncoder* encoder_;
+ absl::optional<EncoderSettings> encoder_settings_;
VideoStreamEncoderObserver* const encoder_stats_observer_;
};
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 96d0d4d..fa11918 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -23,6 +23,7 @@
#include "api/video/video_bitrate_allocator_factory.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/resource_adaptation_module_interface.h"
#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
#include "modules/video_coding/include/video_codec_initializer.h"
#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
@@ -424,7 +425,6 @@
(!encoder_ || encoder_config_.video_format != config.video_format ||
max_data_payload_length_ != max_data_payload_length);
encoder_config_ = std::move(config);
- resource_adaptation_module_->SetEncoderConfig(encoder_config_.Copy());
max_data_payload_length_ = max_data_payload_length;
pending_encoder_reconfiguration_ = true;
@@ -498,7 +498,6 @@
encoder_ = settings_.encoder_factory->CreateVideoEncoder(
encoder_config_.video_format);
- resource_adaptation_module_->SetEncoder(encoder_.get());
// TODO(nisse): What to do if creating the encoder fails? Crash,
// or just discard incoming frames?
RTC_CHECK(encoder_);
@@ -588,7 +587,6 @@
// Make sure the start bit rate is sane...
RTC_DCHECK_LE(codec.startBitrate, 1000000);
max_framerate_ = codec.maxFramerate;
- resource_adaptation_module_->SetCodecMaxFrameRate(max_framerate_);
// Inform source about max configured framerate.
int max_framerate = 0;
@@ -632,6 +630,9 @@
}
send_codec_ = codec;
+ resource_adaptation_module_->SetEncoderSettings(EncoderSettings(
+ encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_));
+
encoder_switch_experiment_.SetCodec(send_codec_.codecType);
quality_rampup_experiment_.SetMaxBitrate(
last_frame_info_->width * last_frame_info_->height, codec.maxBitrate);
@@ -1180,6 +1181,8 @@
}
if (encoder_info_ != info) {
+ resource_adaptation_module_->SetEncoderSettings(EncoderSettings(
+ encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_));
RTC_LOG(LS_INFO) << "Encoder settings changed from "
<< encoder_info_.ToString() << " to " << info.ToString();
}
