Throttle frame-rate In VP8 encoder in steady state for screenshare
If minQP is reached and encoder undershoot consistently, we consider the
quality good enough and throttle encode frame rate.
Bug: webrtc:10310
Change-Id: Ifd07280040dd67ef6e544efdd4619d47bff951e8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/125461
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27003}diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
index e8a6d25..d7cd834 100644
--- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
+++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
@@ -32,6 +32,7 @@
#include "modules/video_coding/utility/simulcast_utility.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/field_trial_parser.h"
+#include "rtc_base/experiments/field_trial_units.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/field_trial.h"
#include "third_party/libyuv/include/libyuv/scale.h"
@@ -53,6 +54,9 @@
constexpr int kTokenPartitions = VP8_ONE_TOKENPARTITION;
constexpr uint32_t kVp832ByteAlign = 32u;
+constexpr int kRtpTicksPerSecond = 90000;
+constexpr int kRtpTicksPerMs = kRtpTicksPerSecond / 1000;
+
// VP8 denoiser states.
enum denoiserState : uint32_t {
kDenoiserOff,
@@ -173,7 +177,11 @@
cpu_speed_default_(-6),
number_of_cores_(0),
rc_max_intra_target_(0),
- key_frame_request_(kMaxSimulcastStreams, false) {
+ key_frame_request_(kMaxSimulcastStreams, false),
+ variable_framerate_experiment_(ParseVariableFramerateConfig(
+ "WebRTC-VP8VariableFramerateScreenshare")),
+ framerate_controller_(variable_framerate_experiment_.framerate_limit),
+ num_steady_state_frames_(0) {
temporal_layers_.reserve(kMaxSimulcastStreams);
raw_images_.reserve(kMaxSimulcastStreams);
encoded_images_.reserve(kMaxSimulcastStreams);
@@ -392,7 +400,7 @@
}
// setting the time base of the codec
configurations_[0].g_timebase.num = 1;
- configurations_[0].g_timebase.den = 90000;
+ configurations_[0].g_timebase.den = kRtpTicksPerSecond;
configurations_[0].g_lag_in_frames = 0; // 0- no frame lagging
// Set the error resilience mode for temporal layers (but not simulcast).
@@ -690,6 +698,39 @@
return enable_frame_dropping ? 30 : 0;
}
+size_t LibvpxVp8Encoder::SteadyStateSize(int sid, int tid) {
+ const int encoder_id = encoders_.size() - 1 - sid;
+ size_t bitrate_bps;
+ float fps;
+ if (SimulcastUtility::IsConferenceModeScreenshare(codec_) ||
+ configurations_[encoder_id].ts_number_layers <= 1) {
+ // In conference screenshare there's no defined per temporal layer bitrate
+ // and framerate.
+ bitrate_bps = configurations_[encoder_id].rc_target_bitrate * 1000;
+ fps = codec_.maxFramerate;
+ } else {
+ bitrate_bps = configurations_[encoder_id].ts_target_bitrate[tid] * 1000;
+ fps = codec_.maxFramerate /
+ fmax(configurations_[encoder_id].ts_rate_decimator[tid], 1.0);
+ if (tid > 0) {
+ // Layer bitrate and fps are counted as a partial sums.
+ bitrate_bps -=
+ configurations_[encoder_id].ts_target_bitrate[tid - 1] * 1000;
+ fps = codec_.maxFramerate /
+ fmax(configurations_[encoder_id].ts_rate_decimator[tid - 1], 1.0);
+ }
+ }
+
+ if (fps < 1e-9)
+ return 0;
+ return static_cast<size_t>(
+ bitrate_bps / (8 * fps) *
+ (100 -
+ variable_framerate_experiment_.steady_state_undershoot_percentage) /
+ 100 +
+ 0.5);
+}
+
int LibvpxVp8Encoder::Encode(const VideoFrame& frame,
const CodecSpecificInfo* codec_specific_info,
const std::vector<FrameType>* frame_types) {
@@ -701,6 +742,33 @@
if (encoded_complete_callback_ == NULL)
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ bool send_key_frame = false;
+ for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size();
+ ++i) {
+ if (key_frame_request_[i] && send_stream_[i]) {
+ send_key_frame = true;
+ break;
+ }
+ }
+ if (!send_key_frame && frame_types) {
+ for (size_t i = 0; i < frame_types->size() && i < send_stream_.size();
+ ++i) {
+ if ((*frame_types)[i] == kVideoFrameKey && send_stream_[i]) {
+ send_key_frame = true;
+ break;
+ }
+ }
+ }
+
+ if (frame.update_rect().IsEmpty() && num_steady_state_frames_ >= 3 &&
+ !send_key_frame) {
+ if (variable_framerate_experiment_.enabled &&
+ framerate_controller_.DropFrame(frame.timestamp() / kRtpTicksPerMs)) {
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+ framerate_controller_.AddFrame(frame.timestamp() / kRtpTicksPerMs);
+ }
+
rtc::scoped_refptr<I420BufferInterface> input_image =
frame.video_frame_buffer()->ToI420();
// Since we are extracting raw pointers from |input_image| to
@@ -736,23 +804,7 @@
raw_images_[i].stride[VPX_PLANE_V], raw_images_[i].d_w,
raw_images_[i].d_h, libyuv::kFilterBilinear);
}
- bool send_key_frame = false;
- for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size();
- ++i) {
- if (key_frame_request_[i] && send_stream_[i]) {
- send_key_frame = true;
- break;
- }
- }
- if (!send_key_frame && frame_types) {
- for (size_t i = 0; i < frame_types->size() && i < send_stream_.size();
- ++i) {
- if ((*frame_types)[i] == kVideoFrameKey && send_stream_[i]) {
- send_key_frame = true;
- break;
- }
- }
- }
+
vpx_enc_frame_flags_t flags[kMaxSimulcastStreams];
Vp8FrameConfig tl_configs[kMaxSimulcastStreams];
for (size_t i = 0; i < encoders_.size(); ++i) {
@@ -812,7 +864,7 @@
// rate control seems to be off with that setup. Using the average input
// frame rate to calculate an average duration for now.
assert(codec_.maxFramerate > 0);
- uint32_t duration = 90000 / codec_.maxFramerate;
+ uint32_t duration = kRtpTicksPerSecond / codec_.maxFramerate;
int error = WEBRTC_VIDEO_CODEC_OK;
int num_tries = 0;
@@ -924,6 +976,14 @@
encoded_images_[encoder_idx].qp_ = qp_128;
encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx],
&codec_specific, nullptr);
+ const size_t steady_state_size = SteadyStateSize(
+ stream_idx, codec_specific.codecSpecific.VP8.temporalIdx);
+ if (qp_128 > variable_framerate_experiment_.steady_state_qp ||
+ encoded_images_[encoder_idx].size() > steady_state_size) {
+ num_steady_state_frames_ = 0;
+ } else {
+ ++num_steady_state_frames_;
+ }
} else if (!temporal_layers_[stream_idx]
->SupportsEncoderFrameDropping()) {
result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
@@ -989,4 +1049,22 @@
return WEBRTC_VIDEO_CODEC_OK;
}
+// static
+LibvpxVp8Encoder::VariableFramerateExperiment
+LibvpxVp8Encoder::ParseVariableFramerateConfig(std::string group_name) {
+ FieldTrialFlag enabled = FieldTrialFlag("Enabled");
+ FieldTrialParameter<double> framerate_limit("min_fps", 5.0);
+ FieldTrialParameter<int> qp("min_qp", 15);
+ FieldTrialParameter<int> undershoot_percentage("undershoot", 30);
+ ParseFieldTrial({&enabled, &framerate_limit, &qp, &undershoot_percentage},
+ field_trial::FindFullName(group_name));
+ VariableFramerateExperiment config;
+ config.enabled = enabled.Get();
+ config.framerate_limit = framerate_limit.Get();
+ config.steady_state_qp = qp.Get();
+ config.steady_state_undershoot_percentage = undershoot_percentage.Get();
+
+ return config;
+}
+
} // namespace webrtc
diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h
index c8c7b08..11d8de9 100644
--- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h
+++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h
@@ -12,6 +12,7 @@
#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
#include <memory>
+#include <string>
#include <vector>
#include "api/video/encoded_image.h"
@@ -23,6 +24,7 @@
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp8/libvpx_interface.h"
#include "modules/video_coding/include/video_codec_interface.h"
+#include "modules/video_coding/utility/framerate_controller.h"
#include "rtc_base/experiments/cpu_speed_experiment.h"
#include "rtc_base/experiments/rate_control_settings.h"
@@ -83,6 +85,8 @@
uint32_t FrameDropThreshold(size_t spatial_idx) const;
+ size_t SteadyStateSize(int sid, int tid);
+
const std::unique_ptr<LibvpxInterface> libvpx_;
const absl::optional<std::vector<CpuSpeedExperiment::Config>>
@@ -106,6 +110,22 @@
std::vector<vpx_codec_ctx_t> encoders_;
std::vector<vpx_codec_enc_cfg_t> configurations_;
std::vector<vpx_rational_t> downsampling_factors_;
+
+ // Variable frame-rate screencast related fields and methods.
+ const struct VariableFramerateExperiment {
+ bool enabled = false;
+ // Framerate is limited to this value in steady state.
+ float framerate_limit = 5.0;
+ // This qp or below is considered a steady state.
+ int steady_state_qp = 15;
+ // Frames of at least this percentage below ideal for configured bitrate are
+ // considered in a steady state.
+ int steady_state_undershoot_percentage = 30;
+ } variable_framerate_experiment_;
+ static VariableFramerateExperiment ParseVariableFramerateConfig(
+ std::string group_name);
+ FramerateController framerate_controller_;
+ int num_steady_state_frames_;
};
} // namespace webrtc
diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc
index 3316ce1..40096e7 100644
--- a/modules/video_coding/video_codec_initializer.cc
+++ b/modules/video_coding/video_codec_initializer.cc
@@ -86,6 +86,7 @@
kDefaultOutlierFrameSizePercent};
RTC_DCHECK_LE(streams.size(), kMaxSimulcastStreams);
+ int max_framerate = 0;
for (size_t i = 0; i < streams.size(); ++i) {
SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
RTC_DCHECK_GT(streams[i].width, 0);
@@ -105,6 +106,7 @@
sim_stream->width = static_cast<uint16_t>(streams[i].width);
sim_stream->height = static_cast<uint16_t>(streams[i].height);
sim_stream->maxFramerate = streams[i].max_framerate;
+ max_framerate = std::max(max_framerate, streams[i].max_framerate);
sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
@@ -134,8 +136,8 @@
if (video_codec.maxBitrate < kEncoderMinBitrateKbps)
video_codec.maxBitrate = kEncoderMinBitrateKbps;
- RTC_DCHECK_GT(streams[0].max_framerate, 0);
- video_codec.maxFramerate = streams[0].max_framerate;
+ RTC_DCHECK_GT(max_framerate, 0);
+ video_codec.maxFramerate = max_framerate;
// Set codec specific options
if (config.encoder_specific_settings)
diff --git a/video/full_stack_tests.cc b/video/full_stack_tests.cc
index 8d74c61..7d7c745 100644
--- a/video/full_stack_tests.cc
+++ b/video/full_stack_tests.cc
@@ -741,9 +741,12 @@
// All the tests using this constant are disabled on Mac.
const char kScreenshareSimulcastExperiment[] =
"WebRTC-SimulcastScreenshare/Enabled/";
-
// TODO(bugs.webrtc.org/9840): Investigate why is this test flaky on Win/Mac.
#if !defined(WEBRTC_WIN)
+const char kScreenshareSimulcastVariableFramerateExperiment[] =
+ "WebRTC-SimulcastScreenshare/Enabled/"
+ "WebRTC-VP8VariableFramerateScreenshare/"
+ "Enabled,min_fps:5.0,min_qp:15,undershoot:30/";
TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast) {
test::ScopedFieldTrials field_trial(
AppendFieldTrials(kScreenshareSimulcastExperiment));
@@ -773,6 +776,36 @@
false};
fixture->RunWithAnalyzer(screenshare);
}
+
+TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast_Variable_Framerate) {
+ test::ScopedFieldTrials field_trial(
+ AppendFieldTrials(kScreenshareSimulcastVariableFramerateExperiment));
+ auto fixture = CreateVideoQualityTestFixture();
+ ParamsWithLogging screenshare;
+ screenshare.call.send_side_bwe = true;
+ screenshare.screenshare[0] = {true, false, 10};
+ screenshare.video[0] = {true, 1850, 1110, 30, 800000, 2500000,
+ 2500000, false, "VP8", 2, 1, 400000,
+ false, false, false, ""};
+ screenshare.analyzer = {"screenshare_slides_simulcast_variable_framerate",
+ 0.0, 0.0, kFullStackTestDurationSecs};
+ ParamsWithLogging screenshare_params_high;
+ screenshare_params_high.video[0] = {
+ true, 1850, 1110, 60, 600000, 1250000, 1250000, false,
+ "VP8", 2, 0, 400000, false, false, false, ""};
+ VideoQualityTest::Params screenshare_params_low;
+ screenshare_params_low.video[0] = {true, 1850, 1110, 5, 30000, 200000,
+ 1000000, false, "VP8", 2, 0, 400000,
+ false, false, false, ""};
+
+ std::vector<VideoStream> streams = {
+ VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0),
+ VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)};
+ screenshare.ss[0] = {
+ streams, 1, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
+ false};
+ fixture->RunWithAnalyzer(screenshare);
+}
#endif // !defined(WEBRTC_WIN)
#endif // !defined(WEBRTC_MAC)
