Change how alignment requirements are processed
Software fallback wrapper now reports least common multiple of requirements
for two encoders.
SimulcastEncoderAdapter queries actual encoder before InitEncode call
and requests alignment for all layers if simulcast is not supported by
any of the encoders.
Bug: chromium:1084702
Change-Id: Iaed8190737125d447036b6c664b863be72556a5d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/225881
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34466}
diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc
index be79c42..bcce9dc 100644
--- a/api/video_codecs/video_encoder_software_fallback_wrapper.cc
+++ b/api/video_codecs/video_encoder_software_fallback_wrapper.cc
@@ -25,6 +25,7 @@
#include "api/video/video_frame.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
+#include "media/base/video_common.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/utility/simulcast_utility.h"
#include "rtc_base/checks.h"
@@ -417,6 +418,13 @@
EncoderInfo info =
IsFallbackActive() ? fallback_encoder_info : default_encoder_info;
+ info.requested_resolution_alignment = cricket::LeastCommonMultiple(
+ fallback_encoder_info.requested_resolution_alignment,
+ default_encoder_info.requested_resolution_alignment);
+ info.apply_alignment_to_all_simulcast_layers =
+ fallback_encoder_info.apply_alignment_to_all_simulcast_layers ||
+ default_encoder_info.apply_alignment_to_all_simulcast_layers;
+
if (fallback_params_.has_value()) {
const auto settings = (encoder_state_ == EncoderState::kForcedFallback)
? fallback_encoder_info.scaling_settings
diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc
index 3af022a..116f987 100644
--- a/media/engine/simulcast_encoder_adapter.cc
+++ b/media/engine/simulcast_encoder_adapter.cc
@@ -149,9 +149,13 @@
SimulcastEncoderAdapter::EncoderContext::EncoderContext(
std::unique_ptr<VideoEncoder> encoder,
- bool prefer_temporal_support)
+ bool prefer_temporal_support,
+ VideoEncoder::EncoderInfo primary_info,
+ VideoEncoder::EncoderInfo fallback_info)
: encoder_(std::move(encoder)),
- prefer_temporal_support_(prefer_temporal_support) {}
+ prefer_temporal_support_(prefer_temporal_support),
+ primary_info_(std::move(primary_info)),
+ fallback_info_(std::move(fallback_info)) {}
void SimulcastEncoderAdapter::EncoderContext::Release() {
if (encoder_) {
@@ -690,7 +694,7 @@
std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
SimulcastEncoderAdapter::FetchOrCreateEncoderContext(
- bool is_lowest_quality_stream) {
+ bool is_lowest_quality_stream) const {
bool prefer_temporal_support = fallback_encoder_factory_ != nullptr &&
is_lowest_quality_stream &&
prefer_temporal_support_on_base_layer_;
@@ -712,14 +716,20 @@
} else {
std::unique_ptr<VideoEncoder> encoder =
primary_encoder_factory_->CreateVideoEncoder(video_format_);
+ VideoEncoder::EncoderInfo primary_info = encoder->GetEncoderInfo();
+ VideoEncoder::EncoderInfo fallback_info = primary_info;
if (fallback_encoder_factory_ != nullptr) {
+ std::unique_ptr<VideoEncoder> fallback_encoder =
+ fallback_encoder_factory_->CreateVideoEncoder(video_format_);
+ fallback_info = fallback_encoder->GetEncoderInfo();
encoder = CreateVideoEncoderSoftwareFallbackWrapper(
- fallback_encoder_factory_->CreateVideoEncoder(video_format_),
- std::move(encoder), prefer_temporal_support);
+ std::move(fallback_encoder), std::move(encoder),
+ prefer_temporal_support);
}
encoder_context = std::make_unique<SimulcastEncoderAdapter::EncoderContext>(
- std::move(encoder), prefer_temporal_support);
+ std::move(encoder), prefer_temporal_support, primary_info,
+ fallback_info);
}
encoder_context->encoder().RegisterEncodeCompleteCallback(
@@ -789,9 +799,11 @@
void SimulcastEncoderAdapter::OverrideFromFieldTrial(
VideoEncoder::EncoderInfo* info) const {
if (encoder_info_override_.requested_resolution_alignment()) {
- info->requested_resolution_alignment =
- *encoder_info_override_.requested_resolution_alignment();
+ info->requested_resolution_alignment = cricket::LeastCommonMultiple(
+ info->requested_resolution_alignment,
+ *encoder_info_override_.requested_resolution_alignment());
info->apply_alignment_to_all_simulcast_layers =
+ info->apply_alignment_to_all_simulcast_layers ||
encoder_info_override_.apply_alignment_to_all_simulcast_layers();
}
if (!encoder_info_override_.resolution_bitrate_limits().empty()) {
@@ -815,7 +827,34 @@
encoder_info.apply_alignment_to_all_simulcast_layers = false;
encoder_info.supports_native_handle = true;
encoder_info.scaling_settings.thresholds = absl::nullopt;
+
if (stream_contexts_.empty()) {
+ // GetEncoderInfo queried before InitEncode. Only alignment info is needed
+ // to be filled.
+ // Create one encoder and query it.
+
+ std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context =
+ FetchOrCreateEncoderContext(true);
+
+ const VideoEncoder::EncoderInfo& primary_info =
+ encoder_context->PrimaryInfo();
+ const VideoEncoder::EncoderInfo& fallback_info =
+ encoder_context->FallbackInfo();
+
+ encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple(
+ primary_info.requested_resolution_alignment,
+ fallback_info.requested_resolution_alignment);
+
+ encoder_info.apply_alignment_to_all_simulcast_layers =
+ primary_info.apply_alignment_to_all_simulcast_layers ||
+ fallback_info.apply_alignment_to_all_simulcast_layers;
+
+ if (!primary_info.supports_simulcast || !fallback_info.supports_simulcast) {
+ encoder_info.apply_alignment_to_all_simulcast_layers = true;
+ }
+
+ cached_encoder_contexts_.emplace_back(std::move(encoder_context));
+
OverrideFromFieldTrial(&encoder_info);
return encoder_info;
}
@@ -825,7 +864,6 @@
for (size_t i = 0; i < stream_contexts_.size(); ++i) {
VideoEncoder::EncoderInfo encoder_impl_info =
stream_contexts_[i].encoder().GetEncoderInfo();
-
if (i == 0) {
// Encoder name indicates names of all sub-encoders.
encoder_info.implementation_name += " (";
@@ -864,7 +902,12 @@
encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple(
encoder_info.requested_resolution_alignment,
encoder_impl_info.requested_resolution_alignment);
- if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) {
+ // request alignment on all layers if any of the encoders may need it, or
+ // if any non-top layer encoder requests a non-trivial alignment.
+ if (encoder_impl_info.apply_alignment_to_all_simulcast_layers ||
+ (encoder_impl_info.requested_resolution_alignment > 1 &&
+ (codec_.simulcastStream[i].height < codec_.height ||
+ codec_.simulcastStream[i].width < codec_.width))) {
encoder_info.apply_alignment_to_all_simulcast_layers = true;
}
}
diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h
index 2cb29ed..07e3ccd 100644
--- a/media/engine/simulcast_encoder_adapter.h
+++ b/media/engine/simulcast_encoder_adapter.h
@@ -71,16 +71,24 @@
class EncoderContext {
public:
EncoderContext(std::unique_ptr<VideoEncoder> encoder,
- bool prefer_temporal_support);
+ bool prefer_temporal_support,
+ VideoEncoder::EncoderInfo primary_info,
+ VideoEncoder::EncoderInfo fallback_info);
EncoderContext& operator=(EncoderContext&&) = delete;
VideoEncoder& encoder() { return *encoder_; }
bool prefer_temporal_support() { return prefer_temporal_support_; }
void Release();
+ const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; }
+
+ const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; }
+
private:
std::unique_ptr<VideoEncoder> encoder_;
bool prefer_temporal_support_;
+ const VideoEncoder::EncoderInfo primary_info_;
+ const VideoEncoder::EncoderInfo fallback_info_;
};
class StreamContext : public EncodedImageCallback {
@@ -138,8 +146,11 @@
void DestroyStoredEncoders();
+ // This method creates encoder. May reuse previously created encoders from
+ // |cached_encoder_contexts_|. It's const because it's used from
+ // const GetEncoderInfo().
std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext(
- bool is_lowest_quality_stream);
+ bool is_lowest_quality_stream) const;
webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec,
int stream_idx,
@@ -169,9 +180,11 @@
// Used for checking the single-threaded access of the encoder interface.
RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_;
- // Store encoders in between calls to Release and InitEncode, so they don't
- // have to be recreated. Remaining encoders are destroyed by the destructor.
- std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
+ // Store previously created and released encoders , so they don't have to be
+ // recreated. Remaining encoders are destroyed by the destructor.
+ // Marked as |mutable| becuase we may need to temporarily create encoder in
+ // GetEncoderInfo(), which is const.
+ mutable std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
const bool boost_base_layer_quality_;
diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc
index a74a2c3..48e005f 100644
--- a/media/engine/simulcast_encoder_adapter_unittest.cc
+++ b/media/engine/simulcast_encoder_adapter_unittest.cc
@@ -900,8 +900,6 @@
}
TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) {
- EXPECT_EQ("SimulcastEncoderAdapter",
- adapter_->GetEncoderInfo().implementation_name);
SimulcastTestFixtureImpl::DefaultSettings(
&codec_, static_cast<const int*>(kTestTemporalLayerProfile),
kVideoCodecVP8);
@@ -910,6 +908,8 @@
encoder_names.push_back("codec2");
encoder_names.push_back("codec3");
helper_->factory()->SetEncoderNames(encoder_names);
+ EXPECT_EQ("SimulcastEncoderAdapter",
+ adapter_->GetEncoderInfo().implementation_name);
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)",
adapter_->GetEncoderInfo().implementation_name);
