Allow encoders to fall back dynamically to software.
Like video_decoder.cc, a call to Encode that returns
WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE will trigger an attempted fallback
to a built-in software encoder. Initialization information, along with
any rate and channel parameter info, will be replayed on the software
encoder and then the frame (that cause the fallback) will be immediately
replayed for the software encoder.
Also modified the existing behavior to Release() the "real" encoder even
if a fallback encoder exists. That seems like the correct behavior.
BUG=webrtc:2920
Review URL: https://codereview.webrtc.org/1328863002
Cr-Commit-Position: refs/heads/master@{#10368}
diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc
index 3e6ce63..d116a1e 100644
--- a/webrtc/video/video_encoder.cc
+++ b/webrtc/video/video_encoder.cc
@@ -50,34 +50,67 @@
VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
VideoCodecType codec_type,
webrtc::VideoEncoder* encoder)
- : encoder_type_(CodecToEncoderType(codec_type)),
+ : rates_set_(false),
+ channel_parameters_set_(false),
+ encoder_type_(CodecToEncoderType(codec_type)),
encoder_(encoder),
- callback_(nullptr) {
+ callback_(nullptr) {}
+
+bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() {
+ RTC_CHECK(encoder_type_ != kUnsupportedCodec)
+ << "Encoder requesting fallback to codec not supported in software.";
+ fallback_encoder_.reset(VideoEncoder::Create(encoder_type_));
+ if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_,
+ max_payload_size_) !=
+ WEBRTC_VIDEO_CODEC_OK) {
+ LOG(LS_ERROR) << "Failed to initialize software-encoder fallback.";
+ fallback_encoder_->Release();
+ fallback_encoder_.reset();
+ return false;
+ }
+ // Replay callback, rates, and channel parameters.
+ if (callback_)
+ fallback_encoder_->RegisterEncodeCompleteCallback(callback_);
+ if (rates_set_)
+ fallback_encoder_->SetRates(bitrate_, framerate_);
+ if (channel_parameters_set_)
+ fallback_encoder_->SetChannelParameters(packet_loss_, rtt_);
+
+ // Since we're switching to the fallback encoder, Release the real encoder. It
+ // may be re-initialized via InitEncode later, and it will continue to get
+ // Set calls for rates and channel parameters in the meantime.
+ encoder_->Release();
+ return true;
}
int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
const VideoCodec* codec_settings,
int32_t number_of_cores,
size_t max_payload_size) {
+ // Store settings, in case we need to dynamically switch to the fallback
+ // encoder after a failed Encode call.
+ codec_settings_ = *codec_settings;
+ number_of_cores_ = number_of_cores;
+ max_payload_size_ = max_payload_size;
+ // Clear stored rate/channel parameters.
+ rates_set_ = false;
+ channel_parameters_set_ = false;
+
int32_t ret =
encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size);
if (ret == WEBRTC_VIDEO_CODEC_OK || encoder_type_ == kUnsupportedCodec) {
+ if (fallback_encoder_)
+ fallback_encoder_->Release();
fallback_encoder_.reset();
if (callback_)
encoder_->RegisterEncodeCompleteCallback(callback_);
return ret;
}
// Try to instantiate software codec.
- fallback_encoder_.reset(VideoEncoder::Create(encoder_type_));
- if (fallback_encoder_->InitEncode(codec_settings, number_of_cores,
- max_payload_size) ==
- WEBRTC_VIDEO_CODEC_OK) {
- if (callback_)
- fallback_encoder_->RegisterEncodeCompleteCallback(callback_);
+ if (InitFallbackEncoder()) {
return WEBRTC_VIDEO_CODEC_OK;
}
- // Software encoder failed, reset and use original return code.
- fallback_encoder_.reset();
+ // Software encoder failed, use original return code.
return ret;
}
@@ -91,6 +124,9 @@
}
int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
+ // If the fallback_encoder_ is non-null, it means it was created via
+ // InitFallbackEncoder which has Release()d encoder_, so we should only ever
+ // need to Release() whichever one is active.
if (fallback_encoder_)
return fallback_encoder_->Release();
return encoder_->Release();
@@ -102,12 +138,21 @@
const std::vector<FrameType>* frame_types) {
if (fallback_encoder_)
return fallback_encoder_->Encode(frame, codec_specific_info, frame_types);
- return encoder_->Encode(frame, codec_specific_info, frame_types);
+ int32_t ret = encoder_->Encode(frame, codec_specific_info, frame_types);
+ // If requested, try a software fallback.
+ if (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE && InitFallbackEncoder()) {
+ // Fallback was successful, so start using it with this frame.
+ return fallback_encoder_->Encode(frame, codec_specific_info, frame_types);
+ }
+ return ret;
}
int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters(
uint32_t packet_loss,
int64_t rtt) {
+ channel_parameters_set_ = true;
+ packet_loss_ = packet_loss;
+ rtt_ = rtt;
int32_t ret = encoder_->SetChannelParameters(packet_loss, rtt);
if (fallback_encoder_)
return fallback_encoder_->SetChannelParameters(packet_loss, rtt);
@@ -116,6 +161,9 @@
int32_t VideoEncoderSoftwareFallbackWrapper::SetRates(uint32_t bitrate,
uint32_t framerate) {
+ rates_set_ = true;
+ bitrate_ = bitrate;
+ framerate_ = framerate;
int32_t ret = encoder_->SetRates(bitrate, framerate);
if (fallback_encoder_)
return fallback_encoder_->SetRates(bitrate, framerate);
diff --git a/webrtc/video/video_encoder_unittest.cc b/webrtc/video/video_encoder_unittest.cc
index 5f2c37a..f848356 100644
--- a/webrtc/video/video_encoder_unittest.cc
+++ b/webrtc/video/video_encoder_unittest.cc
@@ -15,6 +15,8 @@
namespace webrtc {
+const int kWidth = 320;
+const int kHeight = 240;
const size_t kMaxPayloadSize = 800;
class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
@@ -34,7 +36,7 @@
const CodecSpecificInfo* codec_specific_info,
const std::vector<FrameType>* frame_types) override {
++encode_count_;
- return WEBRTC_VIDEO_CODEC_OK;
+ return encode_return_code_;
}
int32_t RegisterEncodeCompleteCallback(
@@ -67,6 +69,7 @@
int init_encode_count_ = 0;
int32_t init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
+ int32_t encode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
int encode_count_ = 0;
EncodedImageCallback* encode_complete_callback_ = nullptr;
int release_count_ = 0;
@@ -87,6 +90,8 @@
};
void UtilizeFallbackEncoder();
+ void FallbackFromEncodeRequest();
+ void EncodeFrame();
FakeEncodedImageCallback callback_;
CountingFakeEncoder fake_encoder_;
@@ -95,9 +100,22 @@
VideoFrame frame_;
};
+void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame() {
+ frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, (kWidth + 1) / 2,
+ (kWidth + 1) / 2);
+ memset(frame_.buffer(webrtc::kYPlane), 16,
+ frame_.allocated_size(webrtc::kYPlane));
+ memset(frame_.buffer(webrtc::kUPlane), 128,
+ frame_.allocated_size(webrtc::kUPlane));
+ memset(frame_.buffer(webrtc::kVPlane), 128,
+ frame_.allocated_size(webrtc::kVPlane));
+
+ std::vector<FrameType> types(1, kKeyFrame);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ fallback_wrapper_.Encode(frame_, nullptr, &types));
+}
+
void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() {
- static const int kWidth = 320;
- static const int kHeight = 240;
fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_);
EXPECT_EQ(&callback_, fake_encoder_.encode_complete_callback_);
@@ -111,20 +129,31 @@
fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize));
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30));
- frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, (kWidth + 1) / 2,
- (kWidth + 1) / 2);
- memset(frame_.buffer(webrtc::kYPlane), 16,
- frame_.allocated_size(webrtc::kYPlane));
- memset(frame_.buffer(webrtc::kUPlane), 128,
- frame_.allocated_size(webrtc::kUPlane));
- memset(frame_.buffer(webrtc::kVPlane), 128,
- frame_.allocated_size(webrtc::kVPlane));
+ int callback_count = callback_.callback_count_;
+ int encode_count = fake_encoder_.encode_count_;
+ EncodeFrame();
+ EXPECT_EQ(encode_count, fake_encoder_.encode_count_);
+ EXPECT_EQ(callback_count + 1, callback_.callback_count_);
+}
- std::vector<FrameType> types(1, kKeyFrame);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
- fallback_wrapper_.Encode(frame_, nullptr, &types));
- EXPECT_EQ(0, fake_encoder_.encode_count_);
- EXPECT_GT(callback_.callback_count_, 0);
+void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() {
+ fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_);
+ codec_.codecType = kVideoCodecVP8;
+ codec_.maxFramerate = 30;
+ codec_.width = kWidth;
+ codec_.height = kHeight;
+ fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30));
+ EXPECT_EQ(1, fake_encoder_.init_encode_count_);
+
+ // Have the non-fallback encoder request a software fallback.
+ fake_encoder_.encode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+ int callback_count = callback_.callback_count_;
+ int encode_count = fake_encoder_.encode_count_;
+ EncodeFrame();
+ // Single encode request, which returned failure.
+ EXPECT_EQ(encode_count + 1, fake_encoder_.encode_count_);
+ EXPECT_EQ(callback_count + 1, callback_.callback_count_);
}
TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InitializesEncoder) {
@@ -133,22 +162,35 @@
EXPECT_EQ(1, fake_encoder_.init_encode_count_);
}
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest, EncodeRequestsFallback) {
+ FallbackFromEncodeRequest();
+ // After fallback, further encodes shouldn't hit the fake encoder.
+ int encode_count = fake_encoder_.encode_count_;
+ EncodeFrame();
+ EXPECT_EQ(encode_count, fake_encoder_.encode_count_);
+}
+
TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanUtilizeFallbackEncoder) {
UtilizeFallbackEncoder();
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
}
TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
- InternalEncoderNotReleasedDuringFallback) {
- UtilizeFallbackEncoder();
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+ InternalEncoderReleasedDuringFallback) {
EXPECT_EQ(0, fake_encoder_.release_count_);
+ UtilizeFallbackEncoder();
+ EXPECT_EQ(1, fake_encoder_.release_count_);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+ // No extra release when the fallback is released.
+ EXPECT_EQ(1, fake_encoder_.release_count_);
}
TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
InternalEncoderNotEncodingDuringFallback) {
UtilizeFallbackEncoder();
- EXPECT_EQ(0, fake_encoder_.encode_count_);
+ int encode_count = fake_encoder_.encode_count_;
+ EncodeFrame();
+ EXPECT_EQ(encode_count, fake_encoder_.encode_count_);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
}
diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h
index 0858a71..609c073 100644
--- a/webrtc/video_encoder.h
+++ b/webrtc/video_encoder.h
@@ -156,6 +156,24 @@
bool SupportsNativeHandle() const override;
private:
+ bool InitFallbackEncoder();
+
+ // Settings used in the last InitEncode call and used if a dynamic fallback to
+ // software is required.
+ VideoCodec codec_settings_;
+ int32_t number_of_cores_;
+ size_t max_payload_size_;
+
+ // The last bitrate/framerate set, and a flag for noting they are set.
+ bool rates_set_;
+ uint32_t bitrate_;
+ uint32_t framerate_;
+
+ // The last channel parameters set, and a flag for noting they are set.
+ bool channel_parameters_set_;
+ uint32_t packet_loss_;
+ int64_t rtt_;
+
const EncoderType encoder_type_;
webrtc::VideoEncoder* const encoder_;
