Allow software fallback on lowest simulcast stream for temporal support
Bug: webrtc:11324
Change-Id: Ie505be0cda74c0444065d86c3727671c62bd4842
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167527
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30437}
diff --git a/api/test/mock_video_encoder.h b/api/test/mock_video_encoder.h
index 65de14f..34c038a 100644
--- a/api/test/mock_video_encoder.h
+++ b/api/test/mock_video_encoder.h
@@ -51,6 +51,10 @@
MOCK_METHOD0(Release, int32_t());
MOCK_METHOD0(Reset, int32_t());
MOCK_METHOD1(SetRates, void(const RateControlParameters& parameters));
+ MOCK_METHOD1(OnPacketLossRateUpdate, void(float packet_loss_rate));
+ MOCK_METHOD1(OnRttUpdate, void(int64_t rtt_ms));
+ MOCK_METHOD1(OnLossNotification,
+ void(const LossNotification& loss_notification));
MOCK_CONST_METHOD0(GetEncoderInfo, EncoderInfo(void));
};
diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn
index 5a16e6b..a3072eb 100644
--- a/api/video_codecs/BUILD.gn
+++ b/api/video_codecs/BUILD.gn
@@ -137,6 +137,7 @@
"../../media:rtc_h264_profile_id",
"../../media:rtc_media_base",
"../../modules/video_coding:video_codec_interface",
+ "../../modules/video_coding:video_coding_utility",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../rtc_base/system:rtc_export",
diff --git a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
index 574bc6f..ba13d92 100644
--- a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
+++ b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
@@ -40,6 +40,7 @@
#include "test/gtest.h"
namespace webrtc {
+using ::testing::_;
using ::testing::Return;
namespace {
@@ -76,6 +77,17 @@
info.has_internal_source = internal_source;
return info;
}
+
+class FakeEncodedImageCallback : public EncodedImageCallback {
+ public:
+ Result OnEncodedImage(const EncodedImage& encoded_image,
+ const CodecSpecificInfo* codec_specific_info,
+ const RTPFragmentationHeader* fragmentation) override {
+ ++callback_count_;
+ return Result(Result::OK, callback_count_);
+ }
+ int callback_count_ = 0;
+};
} // namespace
class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
@@ -86,9 +98,11 @@
const std::string& field_trials)
: override_field_trials_(field_trials),
fake_encoder_(new CountingFakeEncoder()),
+ wrapper_initialized_(false),
fallback_wrapper_(CreateVideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<VideoEncoder>(VP8Encoder::Create()),
- std::unique_ptr<VideoEncoder>(fake_encoder_))) {}
+ std::unique_ptr<VideoEncoder>(fake_encoder_),
+ false)) {}
class CountingFakeEncoder : public VideoEncoder {
public:
@@ -125,9 +139,7 @@
return WEBRTC_VIDEO_CODEC_OK;
}
- void SetRates(const RateControlParameters& parameters) override {
- ++set_rates_count_;
- }
+ void SetRates(const RateControlParameters& parameters) override {}
EncoderInfo GetEncoderInfo() const override {
++supports_native_handle_count_;
@@ -144,23 +156,11 @@
int encode_count_ = 0;
EncodedImageCallback* encode_complete_callback_ = nullptr;
int release_count_ = 0;
- int set_rates_count_ = 0;
mutable int supports_native_handle_count_ = 0;
bool supports_native_handle_ = false;
};
- class FakeEncodedImageCallback : public EncodedImageCallback {
- public:
- Result OnEncodedImage(
- const EncodedImage& encoded_image,
- const CodecSpecificInfo* codec_specific_info,
- const RTPFragmentationHeader* fragmentation) override {
- ++callback_count_;
- return Result(Result::OK, callback_count_);
- }
- int callback_count_ = 0;
- };
-
+ void InitEncode();
void UtilizeFallbackEncoder();
void FallbackFromEncodeRequest();
void EncodeFrame();
@@ -174,6 +174,7 @@
FakeEncodedImageCallback callback_;
// |fake_encoder_| is owned and released by |fallback_wrapper_|.
CountingFakeEncoder* fake_encoder_;
+ bool wrapper_initialized_;
std::unique_ptr<VideoEncoder> fallback_wrapper_;
VideoCodec codec_ = {};
std::unique_ptr<VideoFrame> frame_;
@@ -199,9 +200,42 @@
EXPECT_EQ(expected_ret, fallback_wrapper_->Encode(*frame_, &types));
}
+void VideoEncoderSoftwareFallbackWrapperTest::InitEncode() {
+ if (!wrapper_initialized_) {
+ fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
+ EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
+ }
+
+ // Register fake encoder as main.
+ codec_.codecType = kVideoCodecVP8;
+ codec_.maxFramerate = kFramerate;
+ codec_.width = kWidth;
+ codec_.height = kHeight;
+ codec_.VP8()->numberOfTemporalLayers = 1;
+ rate_allocator_.reset(new SimulcastRateAllocator(codec_));
+
+ if (wrapper_initialized_) {
+ fallback_wrapper_->Release();
+ }
+
+ fake_encoder_->init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ fallback_wrapper_->InitEncode(&codec_, kSettings));
+
+ if (!wrapper_initialized_) {
+ fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters(
+ rate_allocator_->Allocate(
+ VideoBitrateAllocationParameters(300000, kFramerate)),
+ kFramerate));
+ }
+ wrapper_initialized_ = true;
+}
+
void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() {
- fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
- EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
+ if (!wrapper_initialized_) {
+ fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
+ EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
+ }
// Register with failing fake encoder. Should succeed with VP8 fallback.
codec_.codecType = kVideoCodecVP8;
@@ -211,6 +245,10 @@
codec_.VP8()->numberOfTemporalLayers = 1;
rate_allocator_.reset(new SimulcastRateAllocator(codec_));
+ if (wrapper_initialized_) {
+ fallback_wrapper_->Release();
+ }
+
fake_encoder_->init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
fallback_wrapper_->InitEncode(&codec_, kSettings));
@@ -234,6 +272,9 @@
codec_.height = kHeight;
codec_.VP8()->numberOfTemporalLayers = 1;
rate_allocator_.reset(new SimulcastRateAllocator(codec_));
+ if (wrapper_initialized_) {
+ fallback_wrapper_->Release();
+ }
fallback_wrapper_->InitEncode(&codec_, kSettings);
fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters(
rate_allocator_->Allocate(
@@ -272,11 +313,24 @@
TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
InternalEncoderReleasedDuringFallback) {
+ EXPECT_EQ(0, fake_encoder_->init_encode_count_);
EXPECT_EQ(0, fake_encoder_->release_count_);
+
+ InitEncode();
+
+ EXPECT_EQ(1, fake_encoder_->init_encode_count_);
+ EXPECT_EQ(0, fake_encoder_->release_count_);
+
UtilizeFallbackEncoder();
+
+ // One successful InitEncode(), one failed.
+ EXPECT_EQ(2, fake_encoder_->init_encode_count_);
EXPECT_EQ(1, fake_encoder_->release_count_);
+
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
+
// No extra release when the fallback is released.
+ EXPECT_EQ(2, fake_encoder_->init_encode_count_);
EXPECT_EQ(1, fake_encoder_->release_count_);
}
@@ -292,29 +346,30 @@
TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
CanRegisterCallbackWhileUsingFallbackEncoder) {
+ InitEncode();
+ EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
+
UtilizeFallbackEncoder();
- // Registering an encode-complete callback should still work when fallback
- // encoder is being used.
+
+ // Registering an encode-complete callback will now pass to the fallback
+ // instead of the main encoder.
FakeEncodedImageCallback callback2;
fallback_wrapper_->RegisterEncodeCompleteCallback(&callback2);
- EXPECT_EQ(&callback2, fake_encoder_->encode_complete_callback_);
+ EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
// Encoding a frame using the fallback should arrive at the new callback.
std::vector<VideoFrameType> types(1, VideoFrameType::kVideoFrameKey);
frame_->set_timestamp(frame_->timestamp() + 1000);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Encode(*frame_, &types));
+ EXPECT_EQ(callback2.callback_count_, 1);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
-}
+ // Re-initialize to use the main encoder, the new callback should be in use.
+ InitEncode();
+ EXPECT_EQ(&callback2, fake_encoder_->encode_complete_callback_);
-TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
- SetRatesForwardedDuringFallback) {
- UtilizeFallbackEncoder();
- EXPECT_EQ(1, fake_encoder_->set_rates_count_);
- fallback_wrapper_->SetRates(
- VideoEncoder::RateControlParameters(VideoBitrateAllocation(), 1));
- EXPECT_EQ(2, fake_encoder_->set_rates_count_);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
+ frame_->set_timestamp(frame_->timestamp() + 2000);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Encode(*frame_, &types));
+ EXPECT_EQ(callback2.callback_count_, 2);
}
TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
@@ -372,11 +427,12 @@
}
void TearDown() override {
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
+ if (wrapper_initialized_) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
+ }
}
void ConfigureVp8Codec() {
- fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
codec_.codecType = kVideoCodecVP8;
codec_.maxFramerate = kFramerate;
codec_.width = kWidth;
@@ -390,8 +446,13 @@
void InitEncode(int width, int height) {
codec_.width = width;
codec_.height = height;
+ if (wrapper_initialized_) {
+ fallback_wrapper_->Release();
+ }
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
fallback_wrapper_->InitEncode(&codec_, kSettings));
+ fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
+ wrapper_initialized_ = true;
SetRateAllocation(kBitrateKbps);
}
@@ -494,11 +555,11 @@
EXPECT_EQ(1, fake_encoder_->init_encode_count_);
EncodeFrameAndVerifyLastName("fake-encoder");
- // Re-initialize encoder with valid setting but fallback disabled from now on.
+ // Re-initialize encoder with valid setting.
codec_.VP8()->numberOfTemporalLayers = 1;
InitEncode(kWidth, kHeight);
- EXPECT_EQ(2, fake_encoder_->init_encode_count_);
- EncodeFrameAndVerifyLastName("fake-encoder");
+ EXPECT_EQ(1, fake_encoder_->init_encode_count_);
+ EncodeFrameAndVerifyLastName("libvpx");
}
TEST_F(ForcedFallbackTestEnabled, MultipleStartEndFallback) {
@@ -689,4 +750,247 @@
EXPECT_FALSE(wrapper->GetEncoderInfo().has_internal_source);
}
+class PreferTemporalLayersFallbackTest : public ::testing::Test {
+ public:
+ PreferTemporalLayersFallbackTest() {}
+ void SetUp() override {
+ sw_ = new ::testing::NiceMock<MockVideoEncoder>();
+ sw_info_.implementation_name = "sw";
+ EXPECT_CALL(*sw_, GetEncoderInfo).WillRepeatedly([&]() {
+ return sw_info_;
+ });
+ EXPECT_CALL(*sw_, InitEncode(_, _, _))
+ .WillRepeatedly(Return(WEBRTC_VIDEO_CODEC_OK));
+
+ hw_ = new ::testing::NiceMock<MockVideoEncoder>();
+ hw_info_.implementation_name = "hw";
+ EXPECT_CALL(*hw_, GetEncoderInfo()).WillRepeatedly([&]() {
+ return hw_info_;
+ });
+ EXPECT_CALL(*hw_, InitEncode(_, _, _))
+ .WillRepeatedly(Return(WEBRTC_VIDEO_CODEC_OK));
+
+ wrapper_ = CreateVideoEncoderSoftwareFallbackWrapper(
+ std::unique_ptr<VideoEncoder>(sw_), std::unique_ptr<VideoEncoder>(hw_),
+ /*prefer_temporal_support=*/true);
+
+ codec_settings.codecType = kVideoCodecVP8;
+ codec_settings.maxFramerate = kFramerate;
+ codec_settings.width = kWidth;
+ codec_settings.height = kHeight;
+ codec_settings.numberOfSimulcastStreams = 1;
+ codec_settings.VP8()->numberOfTemporalLayers = 1;
+ }
+
+ protected:
+ void SetSupportsLayers(VideoEncoder::EncoderInfo* info, bool tl_enabled) {
+ info->fps_allocation[0].clear();
+ int num_layers = 1;
+ if (tl_enabled) {
+ num_layers = codec_settings.VP8()->numberOfTemporalLayers;
+ }
+ for (int i = 0; i < num_layers; ++i) {
+ info->fps_allocation[0].push_back(
+ VideoEncoder::EncoderInfo::kMaxFramerateFraction >>
+ (num_layers - i - 1));
+ }
+ }
+
+ VideoCodec codec_settings;
+ ::testing::NiceMock<MockVideoEncoder>* sw_;
+ ::testing::NiceMock<MockVideoEncoder>* hw_;
+ VideoEncoder::EncoderInfo sw_info_;
+ VideoEncoder::EncoderInfo hw_info_;
+ std::unique_ptr<VideoEncoder> wrapper_;
+};
+
+TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenLayersNotUsed) {
+ codec_settings.VP8()->numberOfTemporalLayers = 1;
+ SetSupportsLayers(&hw_info_, true);
+ SetSupportsLayers(&sw_info_, true);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ wrapper_->InitEncode(&codec_settings, kSettings));
+ EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+}
+
+TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenLayersSupported) {
+ codec_settings.VP8()->numberOfTemporalLayers = 2;
+ SetSupportsLayers(&hw_info_, true);
+ SetSupportsLayers(&sw_info_, true);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ wrapper_->InitEncode(&codec_settings, kSettings));
+ EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+}
+
+TEST_F(PreferTemporalLayersFallbackTest,
+ UsesFallbackWhenLayersNotSupportedOnMain) {
+ codec_settings.VP8()->numberOfTemporalLayers = 2;
+ SetSupportsLayers(&hw_info_, false);
+ SetSupportsLayers(&sw_info_, true);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ wrapper_->InitEncode(&codec_settings, kSettings));
+ EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw");
+}
+
+TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenNeitherSupportsTemporal) {
+ codec_settings.VP8()->numberOfTemporalLayers = 2;
+ SetSupportsLayers(&hw_info_, false);
+ SetSupportsLayers(&sw_info_, false);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ wrapper_->InitEncode(&codec_settings, kSettings));
+ EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+}
+
+TEST_F(PreferTemporalLayersFallbackTest, PrimesEncoderOnSwitch) {
+ codec_settings.VP8()->numberOfTemporalLayers = 2;
+ // Both support temporal layers, will use main one.
+ SetSupportsLayers(&hw_info_, true);
+ SetSupportsLayers(&sw_info_, true);
+
+ // On first InitEncode most params have no state and will not be
+ // called to update.
+ EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0);
+ EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0);
+
+ EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0);
+ EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0);
+
+ EXPECT_CALL(*hw_, SetRates).Times(0);
+ EXPECT_CALL(*hw_, SetRates).Times(0);
+
+ EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0);
+ EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0);
+
+ EXPECT_CALL(*hw_, OnRttUpdate).Times(0);
+ EXPECT_CALL(*sw_, OnRttUpdate).Times(0);
+
+ EXPECT_CALL(*hw_, OnLossNotification).Times(0);
+ EXPECT_CALL(*sw_, OnLossNotification).Times(0);
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ wrapper_->InitEncode(&codec_settings, kSettings));
+ EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+
+ FakeEncodedImageCallback callback1;
+ class DummyFecControllerOverride : public FecControllerOverride {
+ public:
+ void SetFecAllowed(bool fec_allowed) override {}
+ };
+ DummyFecControllerOverride fec_controller_override1;
+ VideoEncoder::RateControlParameters rate_params1;
+ float packet_loss1 = 0.1;
+ int64_t rtt1 = 1;
+ VideoEncoder::LossNotification lntf1;
+
+ EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback(&callback1));
+ EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0);
+ wrapper_->RegisterEncodeCompleteCallback(&callback1);
+
+ EXPECT_CALL(*hw_, SetFecControllerOverride(&fec_controller_override1));
+ EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0);
+ wrapper_->SetFecControllerOverride(&fec_controller_override1);
+
+ EXPECT_CALL(*hw_, SetRates(rate_params1));
+ EXPECT_CALL(*sw_, SetRates).Times(0);
+ wrapper_->SetRates(rate_params1);
+
+ EXPECT_CALL(*hw_, OnPacketLossRateUpdate(packet_loss1));
+ EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0);
+ wrapper_->OnPacketLossRateUpdate(packet_loss1);
+
+ EXPECT_CALL(*hw_, OnRttUpdate(rtt1));
+ EXPECT_CALL(*sw_, OnRttUpdate).Times(0);
+ wrapper_->OnRttUpdate(rtt1);
+
+ EXPECT_CALL(*hw_, OnLossNotification).Times(1);
+ EXPECT_CALL(*sw_, OnLossNotification).Times(0);
+ wrapper_->OnLossNotification(lntf1);
+
+ // Release and re-init, with fallback to software. This should trigger
+ // the software encoder to be primed with the current state.
+ wrapper_->Release();
+ EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback(&callback1));
+ EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0);
+
+ EXPECT_CALL(*sw_, SetFecControllerOverride(&fec_controller_override1));
+ EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0);
+
+ // Rate control parameters are cleared on InitEncode.
+ EXPECT_CALL(*sw_, SetRates).Times(0);
+ EXPECT_CALL(*hw_, SetRates).Times(0);
+
+ EXPECT_CALL(*sw_, OnPacketLossRateUpdate(packet_loss1));
+ EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0);
+
+ EXPECT_CALL(*sw_, OnRttUpdate(rtt1));
+ EXPECT_CALL(*hw_, OnRttUpdate).Times(0);
+
+ EXPECT_CALL(*sw_, OnLossNotification).Times(1);
+ EXPECT_CALL(*hw_, OnLossNotification).Times(0);
+
+ SetSupportsLayers(&hw_info_, false);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ wrapper_->InitEncode(&codec_settings, kSettings));
+ EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw");
+
+ // Update with all-new params for the software encoder.
+ FakeEncodedImageCallback callback2;
+ DummyFecControllerOverride fec_controller_override2;
+ VideoEncoder::RateControlParameters rate_params2;
+ float packet_loss2 = 0.2;
+ int64_t rtt2 = 2;
+ VideoEncoder::LossNotification lntf2;
+
+ EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback(&callback2));
+ EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0);
+ wrapper_->RegisterEncodeCompleteCallback(&callback2);
+
+ EXPECT_CALL(*sw_, SetFecControllerOverride(&fec_controller_override2));
+ EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0);
+ wrapper_->SetFecControllerOverride(&fec_controller_override2);
+
+ EXPECT_CALL(*sw_, SetRates(rate_params2));
+ EXPECT_CALL(*hw_, SetRates).Times(0);
+ wrapper_->SetRates(rate_params2);
+
+ EXPECT_CALL(*sw_, OnPacketLossRateUpdate(packet_loss2));
+ EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0);
+ wrapper_->OnPacketLossRateUpdate(packet_loss2);
+
+ EXPECT_CALL(*sw_, OnRttUpdate(rtt2));
+ EXPECT_CALL(*hw_, OnRttUpdate).Times(0);
+ wrapper_->OnRttUpdate(rtt2);
+
+ EXPECT_CALL(*sw_, OnLossNotification).Times(1);
+ EXPECT_CALL(*hw_, OnLossNotification).Times(0);
+ wrapper_->OnLossNotification(lntf2);
+
+ // Release and re-init, back to main encoder. This should trigger
+ // the main encoder to be primed with the current state.
+ wrapper_->Release();
+ EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback(&callback2));
+ EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0);
+
+ EXPECT_CALL(*hw_, SetFecControllerOverride(&fec_controller_override2));
+ EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0);
+
+ // Rate control parameters are cleared on InitEncode.
+ EXPECT_CALL(*sw_, SetRates).Times(0);
+ EXPECT_CALL(*hw_, SetRates).Times(0);
+
+ EXPECT_CALL(*hw_, OnPacketLossRateUpdate(packet_loss2));
+ EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0);
+
+ EXPECT_CALL(*hw_, OnRttUpdate(rtt2));
+ EXPECT_CALL(*sw_, OnRttUpdate).Times(0);
+
+ EXPECT_CALL(*hw_, OnLossNotification).Times(1);
+ EXPECT_CALL(*sw_, OnLossNotification).Times(0);
+
+ SetSupportsLayers(&hw_info_, true);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ wrapper_->InitEncode(&codec_settings, kSettings));
+ EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+}
+
} // namespace webrtc
diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc
index fe32741..9edc9b0 100644
--- a/api/video_codecs/video_encoder_software_fallback_wrapper.cc
+++ b/api/video_codecs/video_encoder_software_fallback_wrapper.cc
@@ -15,7 +15,6 @@
#include <cstdio>
#include <memory>
#include <string>
-#include <utility>
#include <vector>
#include "absl/types/optional.h"
@@ -25,6 +24,7 @@
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/video_coding/include/video_error_codes.h"
+#include "modules/video_coding/utility/simulcast_utility.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/field_trial.h"
@@ -33,52 +33,89 @@
namespace {
+// If forced fallback is allowed, either:
+//
+// 1) The forced fallback is requested if the resolution is less than or equal
+// to |max_pixels_|. The resolution is allowed to be scaled down to
+// |min_pixels_|.
+//
+// 2) The forced fallback is requested if temporal support is preferred and the
+// SW fallback supports temporal layers while the HW encoder does not.
+
+struct ForcedFallbackParams {
+ public:
+ bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const {
+ return enable_resolution_based_switch &&
+ codec.codecType == kVideoCodecVP8 &&
+ codec.numberOfSimulcastStreams <= 1 &&
+ codec.VP8().numberOfTemporalLayers == 1 &&
+ codec.width * codec.height <= max_pixels;
+ }
+
+ bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const {
+ return enable_temporal_based_switch &&
+ SimulcastUtility::NumberOfTemporalLayers(codec, 0) > 1;
+ }
+
+ bool enable_temporal_based_switch = false;
+ bool enable_resolution_based_switch = false;
+ int min_pixels = 320 * 180;
+ int max_pixels = 320 * 240;
+};
+
const char kVp8ForceFallbackEncoderFieldTrial[] =
"WebRTC-VP8-Forced-Fallback-Encoder-v2";
-bool EnableForcedFallback() {
- return field_trial::IsEnabled(kVp8ForceFallbackEncoderFieldTrial);
-}
-
-bool IsForcedFallbackPossible(const VideoCodec& codec_settings) {
- return codec_settings.codecType == kVideoCodecVP8 &&
- codec_settings.numberOfSimulcastStreams <= 1 &&
- codec_settings.VP8().numberOfTemporalLayers == 1;
-}
-
-void GetForcedFallbackParamsFromFieldTrialGroup(int* param_min_pixels,
- int* param_max_pixels,
- int minimum_max_pixels) {
- RTC_DCHECK(param_min_pixels);
- RTC_DCHECK(param_max_pixels);
- std::string group =
+absl::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials(
+ const VideoEncoder& main_encoder) {
+ const std::string field_trial =
webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial);
- if (group.empty())
- return;
+ if (field_trial.find("Enabled") != 0) {
+ return absl::nullopt;
+ }
- int min_pixels;
- int max_pixels;
- int min_bps;
- if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &min_pixels, &max_pixels,
- &min_bps) != 3) {
+ int max_pixels_lower_bound =
+ main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1;
+
+ ForcedFallbackParams params;
+ params.enable_resolution_based_switch = true;
+
+ int min_bps = 0;
+ if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", ¶ms.min_pixels,
+ ¶ms.max_pixels, &min_bps) != 3) {
RTC_LOG(LS_WARNING)
<< "Invalid number of forced fallback parameters provided.";
- return;
- }
- if (min_pixels <= 0 || max_pixels < minimum_max_pixels ||
- max_pixels < min_pixels || min_bps <= 0) {
+ return absl::nullopt;
+ } else if (params.min_pixels <= 0 ||
+ params.max_pixels < max_pixels_lower_bound ||
+ params.max_pixels < params.min_pixels || min_bps <= 0) {
RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided.";
- return;
+ return absl::nullopt;
}
- *param_min_pixels = min_pixels;
- *param_max_pixels = max_pixels;
+
+ return params;
+}
+
+absl::optional<ForcedFallbackParams> GetForcedFallbackParams(
+ bool prefer_temporal_support,
+ const VideoEncoder& main_encoder) {
+ absl::optional<ForcedFallbackParams> params =
+ ParseFallbackParamsFromFieldTrials(main_encoder);
+ if (prefer_temporal_support) {
+ if (!params.has_value()) {
+ params.emplace();
+ }
+ params->enable_temporal_based_switch = prefer_temporal_support;
+ }
+ return params;
}
class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder {
public:
VideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
- std::unique_ptr<webrtc::VideoEncoder> hw_encoder);
+ std::unique_ptr<webrtc::VideoEncoder> hw_encoder,
+ bool prefer_temporal_support);
~VideoEncoderSoftwareFallbackWrapper() override;
void SetFecControllerOverride(
@@ -106,28 +143,28 @@
EncoderInfo GetEncoderInfo() const override;
private:
- bool InitFallbackEncoder();
-
- // If |forced_fallback_possible_| is true:
- // The forced fallback is requested if the resolution is less than or equal to
- // |max_pixels_|. The resolution is allowed to be scaled down to
- // |min_pixels_|.
- class ForcedFallbackParams {
- public:
- bool IsValid(const VideoCodec& codec) const {
- return codec.width * codec.height <= max_pixels_;
- }
-
- bool active_ = false;
- int min_pixels_ = 320 * 180;
- int max_pixels_ = 320 * 240;
- };
-
+ bool InitFallbackEncoder(bool is_forced);
bool TryInitForcedFallbackEncoder();
- bool TryReInitForcedFallbackEncoder();
- void ValidateSettingsForForcedFallback();
- bool IsForcedFallbackActive() const;
- void MaybeModifyCodecForFallback();
+ bool IsFallbackActive() const;
+
+ VideoEncoder* current_encoder() {
+ switch (encoder_state_) {
+ case EncoderState::kUninitialized:
+ RTC_LOG(LS_WARNING)
+ << "Trying to access encoder in uninitialized fallback wrapper.";
+ // Return main encoder to preserve previous behavior.
+ ABSL_FALLTHROUGH_INTENDED;
+ case EncoderState::kMainEncoderUsed:
+ return encoder_.get();
+ case EncoderState::kFallbackDueToFailure:
+ case EncoderState::kForcedFallback:
+ return fallback_encoder_.get();
+ }
+ }
+
+ // Updates encoder with last observed parameters, such as callbacks, rates,
+ // etc.
+ void PrimeEncoder(VideoEncoder* encoder) const;
// Settings used in the last InitEncode call and used if a dynamic fallback to
// software is required.
@@ -137,65 +174,95 @@
// The last rate control settings, if set.
absl::optional<RateControlParameters> rate_control_parameters_;
- // The last channel parameters set, and a flag for noting they are set.
- bool channel_parameters_set_;
- uint32_t packet_loss_;
- int64_t rtt_;
+ // The last channel parameters set.
+ absl::optional<float> packet_loss_;
+ absl::optional<int64_t> rtt_;
+ FecControllerOverride* fec_controller_override_;
+ absl::optional<LossNotification> loss_notification_;
- bool use_fallback_encoder_;
+ enum class EncoderState {
+ kUninitialized,
+ kMainEncoderUsed,
+ kFallbackDueToFailure,
+ kForcedFallback
+ };
+
+ EncoderState encoder_state_;
const std::unique_ptr<webrtc::VideoEncoder> encoder_;
-
const std::unique_ptr<webrtc::VideoEncoder> fallback_encoder_;
+
EncodedImageCallback* callback_;
- bool forced_fallback_possible_;
- ForcedFallbackParams forced_fallback_;
+ const absl::optional<ForcedFallbackParams> fallback_params_;
};
VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
- std::unique_ptr<webrtc::VideoEncoder> hw_encoder)
- : channel_parameters_set_(false),
- packet_loss_(0),
- rtt_(0),
- use_fallback_encoder_(false),
+ std::unique_ptr<webrtc::VideoEncoder> hw_encoder,
+ bool prefer_temporal_support)
+ : fec_controller_override_(nullptr),
+ encoder_state_(EncoderState::kUninitialized),
encoder_(std::move(hw_encoder)),
fallback_encoder_(std::move(sw_encoder)),
callback_(nullptr),
- forced_fallback_possible_(EnableForcedFallback()) {
+ fallback_params_(
+ GetForcedFallbackParams(prefer_temporal_support, *encoder_)) {
RTC_DCHECK(fallback_encoder_);
- if (forced_fallback_possible_) {
- GetForcedFallbackParamsFromFieldTrialGroup(
- &forced_fallback_.min_pixels_, &forced_fallback_.max_pixels_,
- encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame -
- 1); // No HW below.
- }
}
+
VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() =
default;
-bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() {
+void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder(
+ VideoEncoder* encoder) const {
+ RTC_DCHECK(encoder);
+ // Replay callback, rates, and channel parameters.
+ if (callback_) {
+ encoder->RegisterEncodeCompleteCallback(callback_);
+ }
+ if (rate_control_parameters_) {
+ encoder->SetRates(*rate_control_parameters_);
+ }
+ if (rtt_.has_value()) {
+ encoder->OnRttUpdate(rtt_.value());
+ }
+ if (packet_loss_.has_value()) {
+ encoder->OnPacketLossRateUpdate(packet_loss_.value());
+ }
+ if (fec_controller_override_) {
+ encoder->SetFecControllerOverride(fec_controller_override_);
+ }
+ if (loss_notification_.has_value()) {
+ encoder->OnLossNotification(loss_notification_.value());
+ }
+}
+
+bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) {
RTC_LOG(LS_WARNING) << "Encoder falling back to software encoding.";
RTC_DCHECK(encoder_settings_.has_value());
const int ret = fallback_encoder_->InitEncode(&codec_settings_,
encoder_settings_.value());
- use_fallback_encoder_ = (ret == WEBRTC_VIDEO_CODEC_OK);
- if (!use_fallback_encoder_) {
+
+ if (ret != WEBRTC_VIDEO_CODEC_OK) {
RTC_LOG(LS_ERROR) << "Failed to initialize software-encoder fallback.";
fallback_encoder_->Release();
return false;
}
- // Replay callback, rates, and channel parameters.
- if (callback_)
- fallback_encoder_->RegisterEncodeCompleteCallback(callback_);
- if (rate_control_parameters_)
- fallback_encoder_->SetRates(*rate_control_parameters_);
- // Since we're switching to the fallback encoder, Release the real encoder. It
- // may be re-initialized via InitEncode later, and it will continue to get
- // Set calls for rates and channel parameters in the meantime.
- encoder_->Release();
+ if (encoder_state_ == EncoderState::kMainEncoderUsed) {
+ // Since we're switching to the fallback encoder, Release the real encoder.
+ // It may be re-initialized via InitEncode later, and it will continue to
+ // get Set calls for rates and channel parameters in the meantime.
+ encoder_->Release();
+ }
+
+ if (is_forced) {
+ encoder_state_ = EncoderState::kForcedFallback;
+ } else {
+ encoder_state_ = EncoderState::kFallbackDueToFailure;
+ }
+
return true;
}
@@ -204,8 +271,9 @@
// It is important that only one of those would ever interact with the
// |fec_controller_override| at a given time. This is the responsibility
// of |this| to maintain.
- encoder_->SetFecControllerOverride(fec_controller_override);
- fallback_encoder_->SetFecControllerOverride(fec_controller_override);
+
+ fec_controller_override_ = fec_controller_override;
+ current_encoder()->SetFecControllerOverride(fec_controller_override);
}
int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
@@ -217,93 +285,94 @@
encoder_settings_ = settings;
// Clear stored rate/channel parameters.
rate_control_parameters_ = absl::nullopt;
- ValidateSettingsForForcedFallback();
- // Try to reinit forced software codec if it is in use.
- if (TryReInitForcedFallbackEncoder()) {
- return WEBRTC_VIDEO_CODEC_OK;
- }
+ RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized)
+ << "InitEncode() should never be called on an active instance!";
+
// Try to init forced software codec if it should be used.
if (TryInitForcedFallbackEncoder()) {
+ PrimeEncoder(current_encoder());
return WEBRTC_VIDEO_CODEC_OK;
}
- forced_fallback_.active_ = false;
int32_t ret = encoder_->InitEncode(codec_settings, settings);
if (ret == WEBRTC_VIDEO_CODEC_OK) {
- if (use_fallback_encoder_) {
- RTC_LOG(LS_WARNING)
- << "InitEncode OK, no longer using the software fallback encoder.";
- fallback_encoder_->Release();
- use_fallback_encoder_ = false;
- }
- if (callback_)
- encoder_->RegisterEncodeCompleteCallback(callback_);
+ encoder_state_ = EncoderState::kMainEncoderUsed;
+ PrimeEncoder(current_encoder());
return ret;
}
+
// Try to instantiate software codec.
- if (InitFallbackEncoder()) {
+ if (InitFallbackEncoder(/*is_forced=*/false)) {
+ PrimeEncoder(current_encoder());
return WEBRTC_VIDEO_CODEC_OK;
}
- // Software encoder failed, use original return code.
+
+ // Software encoder failed too, use original return code.
+ encoder_state_ = EncoderState::kUninitialized;
return ret;
}
int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
callback_ = callback;
- int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback);
- if (use_fallback_encoder_)
- return fallback_encoder_->RegisterEncodeCompleteCallback(callback);
- return ret;
+ return current_encoder()->RegisterEncodeCompleteCallback(callback);
}
int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
- return use_fallback_encoder_ ? fallback_encoder_->Release()
- : encoder_->Release();
+ if (encoder_state_ == EncoderState::kUninitialized) {
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+ int32_t ret = current_encoder()->Release();
+ encoder_state_ = EncoderState::kUninitialized;
+ return ret;
}
int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) {
- if (use_fallback_encoder_)
- return fallback_encoder_->Encode(frame, frame_types);
- int32_t ret = encoder_->Encode(frame, frame_types);
- // If requested, try a software fallback.
- bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
- if (fallback_requested && InitFallbackEncoder()) {
- // Start using the fallback with this frame.
- return fallback_encoder_->Encode(frame, frame_types);
+ switch (encoder_state_) {
+ case EncoderState::kUninitialized:
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ case EncoderState::kMainEncoderUsed: {
+ int32_t ret = encoder_->Encode(frame, frame_types);
+ // If requested, try a software fallback.
+ bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
+ if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) {
+ // Start using the fallback with this frame.
+ PrimeEncoder(current_encoder());
+ return fallback_encoder_->Encode(frame, frame_types);
+ }
+ // Fallback encoder failed too, return original error code.
+ return ret;
+ }
+ case EncoderState::kFallbackDueToFailure:
+ case EncoderState::kForcedFallback:
+ return fallback_encoder_->Encode(frame, frame_types);
}
- return ret;
}
void VideoEncoderSoftwareFallbackWrapper::SetRates(
const RateControlParameters& parameters) {
rate_control_parameters_ = parameters;
- encoder_->SetRates(parameters);
- if (use_fallback_encoder_)
- fallback_encoder_->SetRates(parameters);
+ return current_encoder()->SetRates(parameters);
}
void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate(
float packet_loss_rate) {
- VideoEncoder* encoder =
- use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
- encoder->OnPacketLossRateUpdate(packet_loss_rate);
+ packet_loss_ = packet_loss_rate;
+ current_encoder()->OnPacketLossRateUpdate(packet_loss_rate);
}
void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) {
- VideoEncoder* encoder =
- use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
- encoder->OnRttUpdate(rtt_ms);
+ rtt_ = rtt_ms;
+ current_encoder()->OnRttUpdate(rtt_ms);
}
void VideoEncoderSoftwareFallbackWrapper::OnLossNotification(
const LossNotification& loss_notification) {
- VideoEncoder* encoder =
- use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
- encoder->OnLossNotification(loss_notification);
+ loss_notification_ = loss_notification;
+ current_encoder()->OnLossNotification(loss_notification);
}
VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
@@ -312,17 +381,17 @@
EncoderInfo default_encoder_info = encoder_->GetEncoderInfo();
EncoderInfo info =
- use_fallback_encoder_ ? fallback_encoder_info : default_encoder_info;
+ IsFallbackActive() ? fallback_encoder_info : default_encoder_info;
- if (forced_fallback_possible_) {
- const auto settings = forced_fallback_.active_
+ if (fallback_params_.has_value()) {
+ const auto settings = (encoder_state_ == EncoderState::kForcedFallback)
? fallback_encoder_info.scaling_settings
: default_encoder_info.scaling_settings;
info.scaling_settings =
settings.thresholds
? VideoEncoder::ScalingSettings(settings.thresholds->low,
settings.thresholds->high,
- forced_fallback_.min_pixels_)
+ fallback_params_->min_pixels)
: VideoEncoder::ScalingSettings::kOff;
} else {
info.scaling_settings = default_encoder_info.scaling_settings;
@@ -331,72 +400,82 @@
return info;
}
-bool VideoEncoderSoftwareFallbackWrapper::IsForcedFallbackActive() const {
- return (forced_fallback_possible_ && use_fallback_encoder_ &&
- forced_fallback_.active_);
+bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const {
+ return encoder_state_ == EncoderState::kForcedFallback ||
+ encoder_state_ == EncoderState::kFallbackDueToFailure;
}
bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() {
- if (!forced_fallback_possible_ || use_fallback_encoder_) {
- return false;
- }
- // Fallback not active.
- if (!forced_fallback_.IsValid(codec_settings_)) {
- return false;
- }
- // Settings valid, try to instantiate software codec.
- RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: "
- << codec_settings_.width << "x" << codec_settings_.height;
- if (!InitFallbackEncoder()) {
- return false;
- }
- forced_fallback_.active_ = true;
- return true;
-}
-
-bool VideoEncoderSoftwareFallbackWrapper::TryReInitForcedFallbackEncoder() {
- if (!IsForcedFallbackActive()) {
+ if (!fallback_params_) {
return false;
}
- // Forced fallback active.
- if (!forced_fallback_.IsValid(codec_settings_)) {
- RTC_LOG(LS_INFO) << "Stop forced SW encoder fallback, max pixels exceeded.";
- return false;
+ RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized);
+
+ if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) {
+ // Settings valid, try to instantiate software codec.
+ RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: "
+ << codec_settings_.width << "x" << codec_settings_.height;
+ return InitFallbackEncoder(/*is_forced=*/true);
}
- // Settings valid, reinitialize the forced fallback encoder.
- RTC_DCHECK(encoder_settings_.has_value());
- if (fallback_encoder_->InitEncode(&codec_settings_,
- encoder_settings_.value()) !=
- WEBRTC_VIDEO_CODEC_OK) {
- RTC_LOG(LS_ERROR) << "Failed to init forced SW encoder fallback.";
- return false;
- }
- return true;
-}
-
-void VideoEncoderSoftwareFallbackWrapper::ValidateSettingsForForcedFallback() {
- if (!forced_fallback_possible_)
- return;
-
- if (!IsForcedFallbackPossible(codec_settings_)) {
- if (IsForcedFallbackActive()) {
- fallback_encoder_->Release();
- use_fallback_encoder_ = false;
+ if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) {
+ // First init main encoder to see if that supports temporal layers.
+ if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) ==
+ WEBRTC_VIDEO_CODEC_OK) {
+ encoder_state_ = EncoderState::kMainEncoderUsed;
}
- RTC_LOG(LS_INFO) << "Disable forced_fallback_possible_ due to settings.";
- forced_fallback_possible_ = false;
+
+ if (encoder_state_ == EncoderState::kMainEncoderUsed &&
+ encoder_->GetEncoderInfo().fps_allocation[0].size() > 1) {
+ // Primary encoder already supports temporal layers, use that instead.
+ return true;
+ }
+
+ // Try to initialize fallback and check if it supports temporal layers.
+ if (fallback_encoder_->InitEncode(&codec_settings_,
+ encoder_settings_.value()) ==
+ WEBRTC_VIDEO_CODEC_OK) {
+ if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() > 1) {
+ // Fallback encoder available and supports temporal layers, use it!
+ if (encoder_state_ == EncoderState::kMainEncoderUsed) {
+ // Main encoder initialized but does not support temporal layers,
+ // release it again.
+ encoder_->Release();
+ }
+ encoder_state_ = EncoderState::kForcedFallback;
+ RTC_LOG(LS_INFO)
+ << "Forced switch to SW encoder due to temporal support.";
+ return true;
+ } else {
+ // Fallback encoder intialization succeeded, but it does not support
+ // temporal layers either - release it.
+ fallback_encoder_->Release();
+ }
+ }
+
+ if (encoder_state_ == EncoderState::kMainEncoderUsed) {
+ // Main encoder already initialized - make use of it.
+ RTC_LOG(LS_INFO)
+ << "Cannot fall back for temporal support since fallback that "
+ "supports is not available. Using main encoder instead.";
+ return true;
+ }
}
+
+ // Neither forced fallback mode supported.
+ return false;
}
} // namespace
std::unique_ptr<VideoEncoder> CreateVideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<VideoEncoder> sw_fallback_encoder,
- std::unique_ptr<VideoEncoder> hw_encoder) {
+ std::unique_ptr<VideoEncoder> hw_encoder,
+ bool prefer_temporal_support) {
return std::make_unique<VideoEncoderSoftwareFallbackWrapper>(
- std::move(sw_fallback_encoder), std::move(hw_encoder));
+ std::move(sw_fallback_encoder), std::move(hw_encoder),
+ prefer_temporal_support);
}
} // namespace webrtc
diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.h b/api/video_codecs/video_encoder_software_fallback_wrapper.h
index fa93ab8..5282dcb 100644
--- a/api/video_codecs/video_encoder_software_fallback_wrapper.h
+++ b/api/video_codecs/video_encoder_software_fallback_wrapper.h
@@ -12,6 +12,7 @@
#define API_VIDEO_CODECS_VIDEO_ENCODER_SOFTWARE_FALLBACK_WRAPPER_H_
#include <memory>
+#include <utility>
#include "api/video_codecs/video_encoder.h"
#include "rtc_base/system/rtc_export.h"
@@ -21,10 +22,25 @@
// Used to wrap external VideoEncoders to provide a fallback option on
// software encoding when a hardware encoder fails to encode a stream due to
// hardware restrictions, such as max resolution.
+// |bool prefer_temporal_support| indicates that if the software fallback
+// encoder supports temporal layers but the hardware encoder does not, a
+// fallback should be forced even if the encoder otherwise works.
RTC_EXPORT std::unique_ptr<VideoEncoder>
CreateVideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<VideoEncoder> sw_fallback_encoder,
- std::unique_ptr<VideoEncoder> hw_encoder);
+ std::unique_ptr<VideoEncoder> hw_encoder,
+ bool prefer_temporal_support);
+
+// Default fallback for call-sites not yet updated with
+// |prefer_temporal_support|.
+// TODO(sprang): Remove when usage is gone.
+RTC_EXPORT inline std::unique_ptr<VideoEncoder>
+CreateVideoEncoderSoftwareFallbackWrapper(
+ std::unique_ptr<VideoEncoder> sw_fallback_encoder,
+ std::unique_ptr<VideoEncoder> hw_encoder) {
+ return CreateVideoEncoderSoftwareFallbackWrapper(
+ std::move(sw_fallback_encoder), std::move(hw_encoder), false);
+}
} // namespace webrtc
diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc
index 300439a..af0d66e 100644
--- a/media/engine/simulcast_encoder_adapter.cc
+++ b/media/engine/simulcast_encoder_adapter.cc
@@ -150,7 +150,9 @@
encoded_complete_callback_(nullptr),
experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()),
boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials()
- .Vp8BoostBaseLayerQuality()) {
+ .Vp8BoostBaseLayerQuality()),
+ prefer_temporal_support_on_base_layer_(field_trial::IsEnabled(
+ "WebRTC-Video-PreferTemporalSupportOnBaseLayer")) {
RTC_DCHECK(primary_factory);
// The adapter is typically created on the worker thread, but operated on
@@ -259,7 +261,9 @@
if (fallback_encoder_factory_ != nullptr) {
encoder = CreateVideoEncoderSoftwareFallbackWrapper(
fallback_encoder_factory_->CreateVideoEncoder(format),
- std::move(encoder));
+ std::move(encoder),
+ i == lowest_resolution_stream_index &&
+ prefer_temporal_support_on_base_layer_);
}
}
diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h
index 32e798e..a4cf863 100644
--- a/media/engine/simulcast_encoder_adapter.h
+++ b/media/engine/simulcast_encoder_adapter.h
@@ -133,6 +133,7 @@
const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
const bool boost_base_layer_quality_;
+ const bool prefer_temporal_support_on_base_layer_;
};
} // namespace webrtc
