blob: 1a683ebf4b0e64277bf472bb837429605b7768ac [file] [log] [blame]
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
#include <stdint.h>
#include <cstdio>
#include <string>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/types/optional.h"
#include "api/fec_controller_override.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
namespace {
const char kVp8ForceFallbackEncoderFieldTrial[] =
"WebRTC-VP8-Forced-Fallback-Encoder-v2";
bool EnableForcedFallback() {
return field_trial::IsEnabled(kVp8ForceFallbackEncoderFieldTrial);
}
bool IsForcedFallbackPossible(const VideoCodec& codec_settings) {
return codec_settings.codecType == kVideoCodecVP8 &&
codec_settings.numberOfSimulcastStreams <= 1 &&
codec_settings.VP8().numberOfTemporalLayers == 1;
}
void GetForcedFallbackParamsFromFieldTrialGroup(int* param_min_pixels,
int* param_max_pixels,
int minimum_max_pixels) {
RTC_DCHECK(param_min_pixels);
RTC_DCHECK(param_max_pixels);
std::string group =
webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial);
if (group.empty())
return;
int min_pixels;
int max_pixels;
int min_bps;
if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &min_pixels, &max_pixels,
&min_bps) != 3) {
RTC_LOG(LS_WARNING)
<< "Invalid number of forced fallback parameters provided.";
return;
}
if (min_pixels <= 0 || max_pixels < minimum_max_pixels ||
max_pixels < min_pixels || min_bps <= 0) {
RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided.";
return;
}
*param_min_pixels = min_pixels;
*param_max_pixels = max_pixels;
}
class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder {
public:
VideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
std::unique_ptr<webrtc::VideoEncoder> hw_encoder);
~VideoEncoderSoftwareFallbackWrapper() override;
void SetFecControllerOverride(
FecControllerOverride* fec_controller_override) override;
int32_t InitEncode(const VideoCodec* codec_settings,
const VideoEncoder::Settings& settings) override;
int32_t RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) override;
int32_t Release() override;
int32_t Encode(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) override;
void OnPacketLossRateUpdate(float packet_loss_rate) override;
void OnRttUpdate(int64_t rtt_ms) override;
void OnLossNotification(const LossNotification& loss_notification) override;
void SetRates(const RateControlParameters& parameters) override;
EncoderInfo GetEncoderInfo() const override;
private:
bool InitFallbackEncoder();
// If |forced_fallback_possible_| is true:
// The forced fallback is requested if the resolution is less than or equal to
// |max_pixels_|. The resolution is allowed to be scaled down to
// |min_pixels_|.
class ForcedFallbackParams {
public:
bool IsValid(const VideoCodec& codec) const {
return codec.width * codec.height <= max_pixels_;
}
bool active_ = false;
int min_pixels_ = 320 * 180;
int max_pixels_ = 320 * 240;
};
bool TryInitForcedFallbackEncoder();
bool TryReInitForcedFallbackEncoder();
void ValidateSettingsForForcedFallback();
bool IsForcedFallbackActive() const;
void MaybeModifyCodecForFallback();
// Settings used in the last InitEncode call and used if a dynamic fallback to
// software is required.
VideoCodec codec_settings_;
absl::optional<VideoEncoder::Settings> encoder_settings_;
// The last rate control settings, if set.
absl::optional<RateControlParameters> rate_control_parameters_;
// The last channel parameters set, and a flag for noting they are set.
bool channel_parameters_set_;
uint32_t packet_loss_;
int64_t rtt_;
bool use_fallback_encoder_;
const std::unique_ptr<webrtc::VideoEncoder> encoder_;
const std::unique_ptr<webrtc::VideoEncoder> fallback_encoder_;
EncodedImageCallback* callback_;
bool forced_fallback_possible_;
ForcedFallbackParams forced_fallback_;
};
VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
std::unique_ptr<webrtc::VideoEncoder> hw_encoder)
: channel_parameters_set_(false),
packet_loss_(0),
rtt_(0),
use_fallback_encoder_(false),
encoder_(std::move(hw_encoder)),
fallback_encoder_(std::move(sw_encoder)),
callback_(nullptr),
forced_fallback_possible_(EnableForcedFallback()) {
RTC_DCHECK(fallback_encoder_);
if (forced_fallback_possible_) {
GetForcedFallbackParamsFromFieldTrialGroup(
&forced_fallback_.min_pixels_, &forced_fallback_.max_pixels_,
encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame -
1); // No HW below.
}
}
VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() =
default;
bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() {
RTC_LOG(LS_WARNING) << "Encoder falling back to software encoding.";
RTC_DCHECK(encoder_settings_.has_value());
const int ret = fallback_encoder_->InitEncode(&codec_settings_,
encoder_settings_.value());
use_fallback_encoder_ = (ret == WEBRTC_VIDEO_CODEC_OK);
if (!use_fallback_encoder_) {
RTC_LOG(LS_ERROR) << "Failed to initialize software-encoder fallback.";
fallback_encoder_->Release();
return false;
}
// Replay callback, rates, and channel parameters.
if (callback_)
fallback_encoder_->RegisterEncodeCompleteCallback(callback_);
if (rate_control_parameters_)
fallback_encoder_->SetRates(*rate_control_parameters_);
// Since we're switching to the fallback encoder, Release the real encoder. It
// may be re-initialized via InitEncode later, and it will continue to get
// Set calls for rates and channel parameters in the meantime.
encoder_->Release();
return true;
}
void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride(
FecControllerOverride* fec_controller_override) {
// It is important that only one of those would ever interact with the
// |fec_controller_override| at a given time. This is the responsibility
// of |this| to maintain.
encoder_->SetFecControllerOverride(fec_controller_override);
fallback_encoder_->SetFecControllerOverride(fec_controller_override);
}
int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
const VideoCodec* codec_settings,
const VideoEncoder::Settings& settings) {
// Store settings, in case we need to dynamically switch to the fallback
// encoder after a failed Encode call.
codec_settings_ = *codec_settings;
encoder_settings_ = settings;
// Clear stored rate/channel parameters.
rate_control_parameters_ = absl::nullopt;
ValidateSettingsForForcedFallback();
// Try to reinit forced software codec if it is in use.
if (TryReInitForcedFallbackEncoder()) {
return WEBRTC_VIDEO_CODEC_OK;
}
// Try to init forced software codec if it should be used.
if (TryInitForcedFallbackEncoder()) {
return WEBRTC_VIDEO_CODEC_OK;
}
forced_fallback_.active_ = false;
int32_t ret = encoder_->InitEncode(codec_settings, settings);
if (ret == WEBRTC_VIDEO_CODEC_OK) {
if (use_fallback_encoder_) {
RTC_LOG(LS_WARNING)
<< "InitEncode OK, no longer using the software fallback encoder.";
fallback_encoder_->Release();
use_fallback_encoder_ = false;
}
if (callback_)
encoder_->RegisterEncodeCompleteCallback(callback_);
return ret;
}
// Try to instantiate software codec.
if (InitFallbackEncoder()) {
return WEBRTC_VIDEO_CODEC_OK;
}
// Software encoder failed, use original return code.
return ret;
}
int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
callback_ = callback;
int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback);
if (use_fallback_encoder_)
return fallback_encoder_->RegisterEncodeCompleteCallback(callback);
return ret;
}
int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
return use_fallback_encoder_ ? fallback_encoder_->Release()
: encoder_->Release();
}
int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) {
if (use_fallback_encoder_)
return fallback_encoder_->Encode(frame, frame_types);
int32_t ret = encoder_->Encode(frame, frame_types);
// If requested, try a software fallback.
bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
if (fallback_requested && InitFallbackEncoder()) {
// Start using the fallback with this frame.
return fallback_encoder_->Encode(frame, frame_types);
}
return ret;
}
void VideoEncoderSoftwareFallbackWrapper::SetRates(
const RateControlParameters& parameters) {
rate_control_parameters_ = parameters;
encoder_->SetRates(parameters);
if (use_fallback_encoder_)
fallback_encoder_->SetRates(parameters);
}
void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate(
float packet_loss_rate) {
VideoEncoder* encoder =
use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
encoder->OnPacketLossRateUpdate(packet_loss_rate);
}
void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) {
VideoEncoder* encoder =
use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
encoder->OnRttUpdate(rtt_ms);
}
void VideoEncoderSoftwareFallbackWrapper::OnLossNotification(
const LossNotification& loss_notification) {
VideoEncoder* encoder =
use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
encoder->OnLossNotification(loss_notification);
}
VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
const {
EncoderInfo fallback_encoder_info = fallback_encoder_->GetEncoderInfo();
EncoderInfo default_encoder_info = encoder_->GetEncoderInfo();
EncoderInfo info =
use_fallback_encoder_ ? fallback_encoder_info : default_encoder_info;
if (forced_fallback_possible_) {
const auto settings = forced_fallback_.active_
? fallback_encoder_info.scaling_settings
: default_encoder_info.scaling_settings;
info.scaling_settings =
settings.thresholds
? VideoEncoder::ScalingSettings(settings.thresholds->low,
settings.thresholds->high,
forced_fallback_.min_pixels_)
: VideoEncoder::ScalingSettings::kOff;
} else {
info.scaling_settings = default_encoder_info.scaling_settings;
}
return info;
}
bool VideoEncoderSoftwareFallbackWrapper::IsForcedFallbackActive() const {
return (forced_fallback_possible_ && use_fallback_encoder_ &&
forced_fallback_.active_);
}
bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() {
if (!forced_fallback_possible_ || use_fallback_encoder_) {
return false;
}
// Fallback not active.
if (!forced_fallback_.IsValid(codec_settings_)) {
return false;
}
// Settings valid, try to instantiate software codec.
RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: "
<< codec_settings_.width << "x" << codec_settings_.height;
if (!InitFallbackEncoder()) {
return false;
}
forced_fallback_.active_ = true;
return true;
}
bool VideoEncoderSoftwareFallbackWrapper::TryReInitForcedFallbackEncoder() {
if (!IsForcedFallbackActive()) {
return false;
}
// Forced fallback active.
if (!forced_fallback_.IsValid(codec_settings_)) {
RTC_LOG(LS_INFO) << "Stop forced SW encoder fallback, max pixels exceeded.";
return false;
}
// Settings valid, reinitialize the forced fallback encoder.
RTC_DCHECK(encoder_settings_.has_value());
if (fallback_encoder_->InitEncode(&codec_settings_,
encoder_settings_.value()) !=
WEBRTC_VIDEO_CODEC_OK) {
RTC_LOG(LS_ERROR) << "Failed to init forced SW encoder fallback.";
return false;
}
return true;
}
void VideoEncoderSoftwareFallbackWrapper::ValidateSettingsForForcedFallback() {
if (!forced_fallback_possible_)
return;
if (!IsForcedFallbackPossible(codec_settings_)) {
if (IsForcedFallbackActive()) {
fallback_encoder_->Release();
use_fallback_encoder_ = false;
}
RTC_LOG(LS_INFO) << "Disable forced_fallback_possible_ due to settings.";
forced_fallback_possible_ = false;
}
}
} // namespace
std::unique_ptr<VideoEncoder> CreateVideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<VideoEncoder> sw_fallback_encoder,
std::unique_ptr<VideoEncoder> hw_encoder) {
return absl::make_unique<VideoEncoderSoftwareFallbackWrapper>(
std::move(sw_fallback_encoder), std::move(hw_encoder));
}
} // namespace webrtc