Implement encoder overshoot detector and rate adjuster.
The overshoot detector uses a simple pacer model to determine an
estimate of how much the encoder is overusing the target bitrate.
This utilization factor can then be adjuster for when configuring the
actual target bitrate.
Spatial layers (simulcast streams) are adjusted separately.
Temporal layers are measured separately, but are combined into a single
utilization factor per spatial layer.
Bug: webrtc:10155
Change-Id: I8ea58dc6c4871e880553d7c22202f11cb2feb216
Reviewed-on: https://webrtc-review.googlesource.com/c/114886
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26573}diff --git a/rtc_base/experiments/rate_control_settings.cc b/rtc_base/experiments/rate_control_settings.cc
index cb6e0de..0f19b78 100644
--- a/rtc_base/experiments/rate_control_settings.cc
+++ b/rtc_base/experiments/rate_control_settings.cc
@@ -121,11 +121,12 @@
ParseHysteresisFactor(key_value_config,
kScreenshareHysteresisFieldTrialname,
kDefaultScreenshareHysteresisFactor)),
- probe_max_allocation_("probe_max_allocation", true) {
+ probe_max_allocation_("probe_max_allocation", true),
+ bitrate_adjuster_("bitrate_adjuster", false) {
ParseFieldTrial(
{&congestion_window_, &congestion_window_pushback_, &pacing_factor_,
&alr_probing_, &trust_vp8_, &trust_vp9_, &video_hysteresis_,
- &screenshare_hysteresis_, &probe_max_allocation_},
+ &screenshare_hysteresis_, &probe_max_allocation_, &bitrate_adjuster_},
key_value_config->Lookup("WebRTC-VideoRateControl"));
}
@@ -206,4 +207,8 @@
return probe_max_allocation_.Get();
}
+bool RateControlSettings::UseEncoderBitrateAdjuster() const {
+ return bitrate_adjuster_.Get();
+}
+
} // namespace webrtc
diff --git a/rtc_base/experiments/rate_control_settings.h b/rtc_base/experiments/rate_control_settings.h
index e7dc868..e40c7aa 100644
--- a/rtc_base/experiments/rate_control_settings.h
+++ b/rtc_base/experiments/rate_control_settings.h
@@ -50,6 +50,7 @@
VideoEncoderConfig::ContentType content_type) const;
bool TriggerProbeOnMaxAllocatedBitrateChange() const;
+ bool UseEncoderBitrateAdjuster() const;
private:
explicit RateControlSettings(
@@ -67,6 +68,7 @@
FieldTrialParameter<double> video_hysteresis_;
FieldTrialParameter<double> screenshare_hysteresis_;
FieldTrialParameter<bool> probe_max_allocation_;
+ FieldTrialParameter<bool> bitrate_adjuster_;
};
} // namespace webrtc
diff --git a/rtc_base/experiments/rate_control_settings_unittest.cc b/rtc_base/experiments/rate_control_settings_unittest.cc
index ae9a192..0d8c376 100644
--- a/rtc_base/experiments/rate_control_settings_unittest.cc
+++ b/rtc_base/experiments/rate_control_settings_unittest.cc
@@ -124,6 +124,20 @@
.TriggerProbeOnMaxAllocatedBitrateChange());
}
+TEST(RateControlSettingsTest, UseEncoderBitrateAdjuster) {
+ // Should be off by default.
+ EXPECT_FALSE(
+ RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster());
+
+ {
+ // Can be turned on via field trial.
+ test::ScopedFieldTrials field_trials(
+ "WebRTC-VideoRateControl/bitrate_adjuster:true/");
+ EXPECT_TRUE(RateControlSettings::ParseFromFieldTrials()
+ .UseEncoderBitrateAdjuster());
+ }
+}
+
} // namespace
} // namespace webrtc
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 9874742..610a2fd 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -169,6 +169,10 @@
# visibility = [ "../api/video:video_stream_encoder_create" ]
sources = [
+ "encoder_bitrate_adjuster.cc",
+ "encoder_bitrate_adjuster.h",
+ "encoder_overshoot_detector.cc",
+ "encoder_overshoot_detector.h",
"overuse_frame_detector.cc",
"overuse_frame_detector.h",
"partial_frame_assembler.cc",
@@ -183,7 +187,9 @@
}
deps = [
+ "../api/units:data_rate",
"../api/video:encoded_image",
+ "../api/video:video_bitrate_allocation",
"../api/video:video_bitrate_allocator",
"../api/video:video_bitrate_allocator_factory",
"../api/video:video_frame",
@@ -205,6 +211,7 @@
"../rtc_base:sequenced_task_checker",
"../rtc_base:timeutils",
"../rtc_base/experiments:quality_scaling_experiment",
+ "../rtc_base/experiments:rate_control_settings",
"../rtc_base/system:fallthrough",
"../rtc_base/task_utils:repeating_task",
"../system_wrappers:field_trial",
@@ -456,7 +463,9 @@
"buffered_frame_decryptor_unittest.cc",
"call_stats_unittest.cc",
"cpu_scaling_tests.cc",
+ "encoder_bitrate_adjuster_unittest.cc",
"encoder_key_frame_callback_unittest.cc",
+ "encoder_overshoot_detector_unittest.cc",
"end_to_end_tests/bandwidth_tests.cc",
"end_to_end_tests/call_operation_tests.cc",
"end_to_end_tests/codec_tests.cc",
@@ -505,6 +514,7 @@
"../api:scoped_refptr",
"../api:simulated_network_api",
"../api/test/video:function_video_factory",
+ "../api/units:data_rate",
"../api/video:builtin_video_bitrate_allocator_factory",
"../api/video:encoded_image",
"../api/video:video_bitrate_allocation",
diff --git a/video/encoder_bitrate_adjuster.cc b/video/encoder_bitrate_adjuster.cc
new file mode 100644
index 0000000..eb6017c
--- /dev/null
+++ b/video/encoder_bitrate_adjuster.cc
@@ -0,0 +1,232 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/encoder_bitrate_adjuster.h"
+
+#include <algorithm>
+
+#include "absl/memory/memory.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/time_utils.h"
+
+namespace webrtc {
+
+constexpr int64_t EncoderBitrateAdjuster::kWindowSizeMs;
+constexpr size_t EncoderBitrateAdjuster::kMinFramesSinceLayoutChange;
+constexpr double EncoderBitrateAdjuster::kDefaultUtilizationFactor;
+
+EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings)
+ : current_total_framerate_fps_(0),
+ frames_since_layout_change_(0),
+ min_bitrates_bps_{} {
+ if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9) {
+ for (size_t si = 0; si < codec_settings.VP9().numberOfSpatialLayers; ++si) {
+ if (codec_settings.spatialLayers[si].active) {
+ min_bitrates_bps_[si] =
+ std::max(codec_settings.minBitrate * 1000,
+ codec_settings.spatialLayers[si].minBitrate * 1000);
+ }
+ }
+ } else {
+ for (size_t si = 0; si < codec_settings.numberOfSimulcastStreams; ++si) {
+ if (codec_settings.simulcastStream[si].active) {
+ min_bitrates_bps_[si] =
+ std::max(codec_settings.minBitrate * 1000,
+ codec_settings.simulcastStream[si].minBitrate * 1000);
+ }
+ }
+ }
+}
+
+EncoderBitrateAdjuster::~EncoderBitrateAdjuster() = default;
+
+VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
+ const VideoBitrateAllocation& bitrate_allocation,
+ int framerate_fps) {
+ current_bitrate_allocation_ = bitrate_allocation;
+ current_total_framerate_fps_ = framerate_fps;
+
+ // First check that overshoot detectors exist, and store per spatial layer
+ // how many active temporal layers we have.
+ size_t active_tls_[kMaxSpatialLayers] = {};
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ active_tls_[si] = 0;
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ // Layer is enabled iff it has both positive bitrate and framerate target.
+ if (bitrate_allocation.GetBitrate(si, ti) > 0 &&
+ current_fps_allocation_[si].size() > ti &&
+ current_fps_allocation_[si][ti] > 0) {
+ ++active_tls_[si];
+ if (!overshoot_detectors_[si][ti]) {
+ overshoot_detectors_[si][ti] =
+ absl::make_unique<EncoderOvershootDetector>(kWindowSizeMs);
+ frames_since_layout_change_ = 0;
+ }
+ } else if (overshoot_detectors_[si][ti]) {
+ // Layer removed, destroy overshoot detector.
+ overshoot_detectors_[si][ti].reset();
+ frames_since_layout_change_ = 0;
+ }
+ }
+ }
+
+ // Next poll the overshoot detectors and populate the adjusted allocation.
+ const int64_t now_ms = rtc::TimeMillis();
+ VideoBitrateAllocation adjusted_allocation;
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ const uint32_t spatial_layer_bitrate_bps =
+ bitrate_allocation.GetSpatialLayerSum(si);
+
+ // Adjustment is done per spatial layer only (not per temporal layer).
+ double utilization_factor;
+ if (frames_since_layout_change_ < kMinFramesSinceLayoutChange) {
+ utilization_factor = kDefaultUtilizationFactor;
+ } else if (active_tls_[si] == 0 || spatial_layer_bitrate_bps == 0) {
+ // No signaled temporal layers, or no bitrate set. Could either be unused
+ // spatial layer or bitrate dynamic mode; pass bitrate through without any
+ // change.
+ utilization_factor = 1.0;
+ } else if (active_tls_[si] == 1) {
+ // A single active temporal layer, this might mean single layer or that
+ // encoder does not support temporal layers. Merge target bitrates for
+ // this spatial layer.
+ RTC_DCHECK(overshoot_detectors_[si][0]);
+ utilization_factor =
+ overshoot_detectors_[si][0]->GetUtilizationFactor(now_ms).value_or(
+ kDefaultUtilizationFactor);
+ } else if (spatial_layer_bitrate_bps > 0) {
+ // Multiple temporal layers enabled for this spatial layer. Update rate
+ // for each of them and make a weighted average of utilization factors,
+ // with bitrate fraction used as weight.
+ // If any layer is missing a utilization factor, fall back to default.
+ utilization_factor = 0.0;
+ for (size_t ti = 0; ti < active_tls_[si]; ++ti) {
+ RTC_DCHECK(overshoot_detectors_[si][ti]);
+ const absl::optional<double> ti_utilization_factor =
+ overshoot_detectors_[si][ti]->GetUtilizationFactor(now_ms);
+ if (!ti_utilization_factor) {
+ utilization_factor = kDefaultUtilizationFactor;
+ break;
+ }
+ const double weight =
+ static_cast<double>(bitrate_allocation.GetBitrate(si, ti)) /
+ spatial_layer_bitrate_bps;
+ utilization_factor += weight * ti_utilization_factor.value();
+ }
+ } else {
+ RTC_NOTREACHED();
+ }
+
+ // Don't boost target bitrate if encoder is under-using.
+ utilization_factor = std::max(utilization_factor, 1.0);
+
+ // Don't reduce encoder target below 50%, in which case the frame dropper
+ // should kick in instead.
+ utilization_factor = std::min(utilization_factor, 2.0);
+
+ if (min_bitrates_bps_[si] > 0 && spatial_layer_bitrate_bps > 0 &&
+ min_bitrates_bps_[si] < spatial_layer_bitrate_bps) {
+ // Make sure rate adjuster doesn't push target bitrate below minimum.
+ utilization_factor = std::min(
+ utilization_factor, static_cast<double>(spatial_layer_bitrate_bps) /
+ min_bitrates_bps_[si]);
+ }
+
+ // Populate the adjusted allocation with determined utilization factor.
+ if (active_tls_[si] == 1 &&
+ spatial_layer_bitrate_bps > bitrate_allocation.GetBitrate(si, 0)) {
+ // Bitrate allocation indicates temporal layer usage, but encoder
+ // does not seem to support it. Pipe all bitrate into a single
+ // overshoot detector.
+ uint32_t adjusted_layer_bitrate_bps = static_cast<uint32_t>(
+ spatial_layer_bitrate_bps / utilization_factor + 0.5);
+ adjusted_allocation.SetBitrate(si, 0, adjusted_layer_bitrate_bps);
+ } else {
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ if (bitrate_allocation.HasBitrate(si, ti)) {
+ uint32_t adjusted_layer_bitrate_bps = static_cast<uint32_t>(
+ bitrate_allocation.GetBitrate(si, ti) / utilization_factor + 0.5);
+ adjusted_allocation.SetBitrate(si, ti, adjusted_layer_bitrate_bps);
+ }
+ }
+ }
+
+ // In case of rounding errors, add bitrate to TL0 until min bitrate
+ // constraint has been met.
+ const uint32_t adjusted_spatial_layer_sum =
+ adjusted_allocation.GetSpatialLayerSum(si);
+ if (spatial_layer_bitrate_bps > 0 &&
+ adjusted_spatial_layer_sum < min_bitrates_bps_[si]) {
+ adjusted_allocation.SetBitrate(si, 0,
+ adjusted_allocation.GetBitrate(si, 0) +
+ min_bitrates_bps_[si] -
+ adjusted_spatial_layer_sum);
+ }
+
+ // Update all detectors with the new adjusted bitrate targets.
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ const uint32_t layer_bitrate_bps = adjusted_allocation.GetBitrate(si, ti);
+ // Overshoot detector may not exist, eg for ScreenshareLayers case.
+ if (layer_bitrate_bps > 0 && overshoot_detectors_[si][ti]) {
+ // Number of frames in this layer alone is not cumulative, so
+ // subtract fps from any low temporal layer.
+ const double fps_fraction =
+ static_cast<double>(
+ current_fps_allocation_[si][ti] -
+ (ti == 0 ? 0 : current_fps_allocation_[si][ti - 1])) /
+ VideoEncoder::EncoderInfo::kMaxFramerateFraction;
+
+ overshoot_detectors_[si][ti]->SetTargetRate(
+ DataRate::bps(layer_bitrate_bps),
+ fps_fraction * current_total_framerate_fps_, now_ms);
+ }
+ }
+ }
+
+ return adjusted_allocation;
+}
+
+void EncoderBitrateAdjuster::OnEncoderInfo(
+ const VideoEncoder::EncoderInfo& encoder_info) {
+ // Copy allocation into current state and re-allocate.
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ current_fps_allocation_[si] = encoder_info.fps_allocation[si];
+ }
+
+ // Trigger re-allocation so that overshoot detectors have correct targets.
+ AdjustRateAllocation(current_bitrate_allocation_,
+ current_total_framerate_fps_);
+}
+
+void EncoderBitrateAdjuster::OnEncodedFrame(const EncodedImage& encoded_image,
+ int temporal_index) {
+ ++frames_since_layout_change_;
+ // Detectors may not exist, for instance if ScreenshareLayers is used.
+ auto& detector =
+ overshoot_detectors_[encoded_image.SpatialIndex().value_or(0)]
+ [temporal_index];
+ if (detector) {
+ detector->OnEncodedFrame(encoded_image.size(), rtc::TimeMillis());
+ }
+}
+
+void EncoderBitrateAdjuster::Reset() {
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ overshoot_detectors_[si][ti].reset();
+ }
+ }
+ // Call AdjustRateAllocation() with the last know bitrate allocation, so that
+ // the appropriate overuse detectors are immediately re-created.
+ AdjustRateAllocation(current_bitrate_allocation_,
+ current_total_framerate_fps_);
+}
+
+} // namespace webrtc
diff --git a/video/encoder_bitrate_adjuster.h b/video/encoder_bitrate_adjuster.h
new file mode 100644
index 0000000..8901ad4
--- /dev/null
+++ b/video/encoder_bitrate_adjuster.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_ENCODER_BITRATE_ADJUSTER_H_
+#define VIDEO_ENCODER_BITRATE_ADJUSTER_H_
+
+#include <memory>
+
+#include "api/video/encoded_image.h"
+#include "api/video/video_bitrate_allocation.h"
+#include "api/video_codecs/video_encoder.h"
+#include "video/encoder_overshoot_detector.h"
+
+namespace webrtc {
+
+class EncoderBitrateAdjuster {
+ public:
+ // Size of sliding window used to track overshoot rate.
+ static constexpr int64_t kWindowSizeMs = 3000;
+ // Minimum number of frames since last layout change required to trust the
+ // overshoot statistics. Otherwise falls back to default utilization.
+ // By layout change, we mean any spatial/temporal layer being either enabled
+ // or disabled.
+ static constexpr size_t kMinFramesSinceLayoutChange = 30;
+ // Default utilization, before reliable metrics are available, is set to 20%
+ // overshoot. This is conservative so that badly misbehaving encoders don't
+ // build too much queue at the very start.
+ static constexpr double kDefaultUtilizationFactor = 1.2;
+
+ explicit EncoderBitrateAdjuster(const VideoCodec& codec_settings);
+ ~EncoderBitrateAdjuster();
+
+ // Adjusts the given rate allocation to make it paceable within the target
+ // rates.
+ VideoBitrateAllocation AdjustRateAllocation(
+ const VideoBitrateAllocation& bitrate_allocation,
+ int framerate_fps);
+
+ // Updated overuse detectors with data about the encoder, specifically about
+ // the temporal layer frame rate allocation.
+ void OnEncoderInfo(const VideoEncoder::EncoderInfo& encoder_info);
+
+ // Updates the overuse detectors according to the encoded image size.
+ void OnEncodedFrame(const EncodedImage& encoded_image, int temporal_index);
+
+ void Reset();
+
+ private:
+ VideoBitrateAllocation current_bitrate_allocation_;
+ int current_total_framerate_fps_;
+ // FPS allocation of temporal layers, per spatial layer. Represented as a Q8
+ // fraction; 0 = 0%, 255 = 100%. See VideoEncoder::EncoderInfo.fps_allocation.
+ absl::InlinedVector<uint8_t, kMaxTemporalStreams>
+ current_fps_allocation_[kMaxSpatialLayers];
+
+ // Frames since layout was changed, mean that any spatial or temporal layer
+ // was either disabled or enabled.
+ size_t frames_since_layout_change_;
+ std::unique_ptr<EncoderOvershootDetector>
+ overshoot_detectors_[kMaxSpatialLayers][kMaxTemporalStreams];
+
+ // Minimum bitrates allowed, per spatial layer.
+ uint32_t min_bitrates_bps_[kMaxSpatialLayers];
+};
+
+} // namespace webrtc
+
+#endif // VIDEO_ENCODER_BITRATE_ADJUSTER_H_
diff --git a/video/encoder_bitrate_adjuster_unittest.cc b/video/encoder_bitrate_adjuster_unittest.cc
new file mode 100644
index 0000000..312fde7
--- /dev/null
+++ b/video/encoder_bitrate_adjuster_unittest.cc
@@ -0,0 +1,380 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/encoder_bitrate_adjuster.h"
+
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "api/units/data_rate.h"
+#include "rtc_base/fake_clock.h"
+#include "rtc_base/numerics/safe_conversions.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+class EncoderBitrateAdjusterTest : public ::testing::Test {
+ public:
+ static constexpr int64_t kWindowSizeMs = 3000;
+ static constexpr int kDefaultBitrateBps = 300000;
+ static constexpr int kDefaultFrameRateFps = 30;
+ EncoderBitrateAdjusterTest()
+ : target_bitrate_(DataRate::bps(kDefaultBitrateBps)),
+ target_framerate_fps_(kDefaultFrameRateFps),
+ tl_pattern_idx_{} {}
+
+ protected:
+ void SetUpAdjuster(size_t num_spatial_layers,
+ size_t num_temporal_layers,
+ bool vp9_svc) {
+ // Initialize some default VideoCodec instance with the given number of
+ // layers.
+ if (vp9_svc) {
+ codec_.codecType = VideoCodecType::kVideoCodecVP9;
+ codec_.numberOfSimulcastStreams = 1;
+ codec_.VP9()->numberOfSpatialLayers = num_spatial_layers;
+ codec_.VP9()->numberOfTemporalLayers = num_temporal_layers;
+ for (size_t si = 0; si < num_spatial_layers; ++si) {
+ codec_.spatialLayers[si].minBitrate = 100 * (1 << si);
+ codec_.spatialLayers[si].targetBitrate = 200 * (1 << si);
+ codec_.spatialLayers[si].maxBitrate = 300 * (1 << si);
+ codec_.spatialLayers[si].active = true;
+ codec_.spatialLayers[si].numberOfTemporalLayers = num_temporal_layers;
+ }
+ } else {
+ codec_.codecType = VideoCodecType::kVideoCodecVP8;
+ codec_.numberOfSimulcastStreams = num_spatial_layers;
+ codec_.VP8()->numberOfTemporalLayers = num_temporal_layers;
+ for (size_t si = 0; si < num_spatial_layers; ++si) {
+ codec_.simulcastStream[si].minBitrate = 100 * (1 << si);
+ codec_.simulcastStream[si].targetBitrate = 200 * (1 << si);
+ codec_.simulcastStream[si].maxBitrate = 300 * (1 << si);
+ codec_.simulcastStream[si].active = true;
+ codec_.simulcastStream[si].numberOfTemporalLayers = num_temporal_layers;
+ }
+ }
+
+ for (size_t si = 0; si < num_spatial_layers; ++si) {
+ encoder_info_.fps_allocation[si].resize(num_temporal_layers);
+ double fraction = 1.0;
+ for (int ti = num_temporal_layers - 1; ti >= 0; --ti) {
+ encoder_info_.fps_allocation[si][ti] = static_cast<uint8_t>(
+ VideoEncoder::EncoderInfo::kMaxFramerateFraction * fraction + 0.5);
+ fraction /= 2.0;
+ }
+ }
+
+ adjuster_ = absl::make_unique<EncoderBitrateAdjuster>(codec_);
+ adjuster_->OnEncoderInfo(encoder_info_);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ }
+
+ void InsertFrames(std::vector<std::vector<double>> utilization_factors,
+ int64_t duration_ms) {
+ constexpr size_t kMaxFrameSize = 100000;
+ uint8_t buffer[kMaxFrameSize];
+
+ const int64_t start_us = rtc::TimeMicros();
+ while (rtc::TimeMicros() <
+ start_us + (duration_ms * rtc::kNumMicrosecsPerMillisec)) {
+ clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec /
+ target_framerate_fps_);
+ for (size_t si = 0; si < NumSpatialLayers(); ++si) {
+ const std::vector<int>& tl_pattern =
+ kTlPatterns[NumTemporalLayers(si) - 1];
+ const size_t ti =
+ tl_pattern[(tl_pattern_idx_[si]++) % tl_pattern.size()];
+
+ uint32_t layer_bitrate_bps =
+ current_adjusted_allocation_.GetBitrate(si, ti);
+ double layer_framerate_fps = target_framerate_fps_;
+ if (encoder_info_.fps_allocation[si].size() > ti) {
+ uint8_t layer_fps_fraction = encoder_info_.fps_allocation[si][ti];
+ if (ti > 0) {
+ // We're interested in the frame rate for this layer only, not
+ // cumulative frame rate.
+ layer_fps_fraction -= encoder_info_.fps_allocation[si][ti - 1];
+ }
+ layer_framerate_fps =
+ (target_framerate_fps_ * layer_fps_fraction) /
+ VideoEncoder::EncoderInfo::kMaxFramerateFraction;
+ }
+ double utilization_factor = 1.0;
+ if (utilization_factors.size() > si &&
+ utilization_factors[si].size() > ti) {
+ utilization_factor = utilization_factors[si][ti];
+ }
+ size_t frame_size_bytes = utilization_factor *
+ (layer_bitrate_bps / 8.0) /
+ layer_framerate_fps;
+
+ EncodedImage image(buffer, 0, kMaxFrameSize);
+ image.set_size(frame_size_bytes);
+ image.SetSpatialIndex(si);
+ adjuster_->OnEncodedFrame(image, ti);
+ }
+ }
+ }
+
+ size_t NumSpatialLayers() const {
+ if (codec_.codecType == VideoCodecType::kVideoCodecVP9) {
+ return codec_.VP9().numberOfSpatialLayers;
+ }
+ return codec_.numberOfSimulcastStreams;
+ }
+
+ size_t NumTemporalLayers(int spatial_index) {
+ if (codec_.codecType == VideoCodecType::kVideoCodecVP9) {
+ return codec_.spatialLayers[spatial_index].numberOfTemporalLayers;
+ }
+ return codec_.simulcastStream[spatial_index].numberOfTemporalLayers;
+ }
+
+ void ExpectNear(const VideoBitrateAllocation& expected_allocation,
+ const VideoBitrateAllocation& actual_allocation,
+ double allowed_error_fraction) {
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ if (expected_allocation.HasBitrate(si, ti)) {
+ EXPECT_TRUE(actual_allocation.HasBitrate(si, ti));
+ uint32_t expected_layer_bitrate_bps =
+ expected_allocation.GetBitrate(si, ti);
+ EXPECT_NEAR(expected_layer_bitrate_bps,
+ actual_allocation.GetBitrate(si, ti),
+ static_cast<uint32_t>(expected_layer_bitrate_bps *
+ allowed_error_fraction));
+ } else {
+ EXPECT_FALSE(actual_allocation.HasBitrate(si, ti));
+ }
+ }
+ }
+ }
+
+ VideoBitrateAllocation MultiplyAllocation(
+ const VideoBitrateAllocation& allocation,
+ double factor) {
+ VideoBitrateAllocation multiplied_allocation;
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ if (allocation.HasBitrate(si, ti)) {
+ multiplied_allocation.SetBitrate(
+ si, ti,
+ static_cast<uint32_t>(factor * allocation.GetBitrate(si, ti) +
+ 0.5));
+ }
+ }
+ }
+ return multiplied_allocation;
+ }
+
+ VideoCodec codec_;
+ VideoEncoder::EncoderInfo encoder_info_;
+ std::unique_ptr<EncoderBitrateAdjuster> adjuster_;
+ VideoBitrateAllocation current_input_allocation_;
+ VideoBitrateAllocation current_adjusted_allocation_;
+ rtc::ScopedFakeClock clock_;
+ DataRate target_bitrate_;
+ double target_framerate_fps_;
+ int tl_pattern_idx_[kMaxSpatialLayers];
+
+ const std::vector<int> kTlPatterns[kMaxTemporalStreams] = {
+ {0},
+ {0, 1},
+ {0, 2, 1, 2},
+ {0, 3, 2, 3, 1, 3, 2, 3}};
+};
+
+TEST_F(EncoderBitrateAdjusterTest, SingleLayerOptimal) {
+ // Single layer, well behaved encoder.
+ current_input_allocation_.SetBitrate(0, 0, 300000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 1, false);
+ InsertFrames({{1.0}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ // Adjusted allocation near input. Allow 1% error margin due to rounding
+ // errors etc.
+ ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, SingleLayerOveruse) {
+ // Single layer, well behaved encoder.
+ current_input_allocation_.SetBitrate(0, 0, 300000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 1, false);
+ InsertFrames({{1.2}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ // Adjusted allocation lowered by 20%.
+ ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.2),
+ current_adjusted_allocation_, 0.01);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, SingleLayerUnderuse) {
+ // Single layer, well behaved encoder.
+ current_input_allocation_.SetBitrate(0, 0, 300000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 1, false);
+ InsertFrames({{0.5}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ // Undershoot, adjusted should exactly match input.
+ ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.00);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersOptimalSize) {
+ // Three temporal layers, 60%/20%/20% bps distro, well behaved encoder.
+ current_input_allocation_.SetBitrate(0, 0, 180000);
+ current_input_allocation_.SetBitrate(0, 1, 60000);
+ current_input_allocation_.SetBitrate(0, 2, 60000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 3, false);
+ InsertFrames({{1.0, 1.0, 1.0}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersOvershoot) {
+ // Three temporal layers, 60%/20%/20% bps distro.
+ // 10% overshoot on all layers.
+ current_input_allocation_.SetBitrate(0, 0, 180000);
+ current_input_allocation_.SetBitrate(0, 1, 60000);
+ current_input_allocation_.SetBitrate(0, 2, 60000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 3, false);
+ InsertFrames({{1.1, 1.1, 1.1}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ // Adjusted allocation lowered by 10%.
+ ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.1),
+ current_adjusted_allocation_, 0.01);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersUndershoot) {
+ // Three temporal layers, 60%/20%/20% bps distro, undershoot all layers.
+ current_input_allocation_.SetBitrate(0, 0, 180000);
+ current_input_allocation_.SetBitrate(0, 1, 60000);
+ current_input_allocation_.SetBitrate(0, 2, 60000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 3, false);
+ InsertFrames({{0.8, 0.8, 0.8}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ // Adjusted allocation identical since we don't boost bitrates.
+ ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.0);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersSkewedOvershoot) {
+ // Three temporal layers, 60%/20%/20% bps distro.
+ // 10% overshoot on base layer, 20% on higher layers.
+ current_input_allocation_.SetBitrate(0, 0, 180000);
+ current_input_allocation_.SetBitrate(0, 1, 60000);
+ current_input_allocation_.SetBitrate(0, 2, 60000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 3, false);
+ InsertFrames({{1.1, 1.2, 1.2}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ // Expected overshoot is weighted by bitrate:
+ // (0.6 * 1.1 + 0.2 * 1.2 + 0.2 * 1.2) = 1.14
+ ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.14),
+ current_adjusted_allocation_, 0.01);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, FourTemporalLayersSkewedOvershoot) {
+ // Three temporal layers, 40%/30%/15%/15% bps distro.
+ // 10% overshoot on base layer, 20% on higher layers.
+ current_input_allocation_.SetBitrate(0, 0, 120000);
+ current_input_allocation_.SetBitrate(0, 1, 90000);
+ current_input_allocation_.SetBitrate(0, 2, 45000);
+ current_input_allocation_.SetBitrate(0, 3, 45000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 4, false);
+ InsertFrames({{1.1, 1.2, 1.2, 1.2}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ // Expected overshoot is weighted by bitrate:
+ // (0.4 * 1.1 + 0.3 * 1.2 + 0.15 * 1.2 + 0.15 * 1.2) = 1.16
+ ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.16),
+ current_adjusted_allocation_, 0.01);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersNonLayeredEncoder) {
+ // Three temporal layers, 60%/20%/20% bps allocation, 10% overshoot,
+ // encoder does not actually support temporal layers.
+ current_input_allocation_.SetBitrate(0, 0, 180000);
+ current_input_allocation_.SetBitrate(0, 1, 60000);
+ current_input_allocation_.SetBitrate(0, 2, 60000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 1, false);
+ InsertFrames({{1.1}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ // Expect the actual 10% overuse to be detected and the allocation to
+ // only contain the one entry.
+ VideoBitrateAllocation expected_allocation;
+ expected_allocation.SetBitrate(
+ 0, 0,
+ static_cast<uint32_t>(current_input_allocation_.get_sum_bps() / 1.10));
+ ExpectNear(expected_allocation, current_adjusted_allocation_, 0.01);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, IgnoredStream) {
+ // Encoder with three temporal layers, but in a mode that does not support
+ // deterministic frame rate. Those are ignored, even if bitrate overshoots.
+ current_input_allocation_.SetBitrate(0, 0, 180000);
+ current_input_allocation_.SetBitrate(0, 1, 60000);
+ target_framerate_fps_ = 30;
+ SetUpAdjuster(1, 1, false);
+ encoder_info_.fps_allocation[0].clear();
+ adjuster_->OnEncoderInfo(encoder_info_);
+
+ InsertFrames({{1.1}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+
+ // Values passed through.
+ ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.00);
+}
+
+TEST_F(EncoderBitrateAdjusterTest, DifferentSpatialOvershoots) {
+ // Two streams, both with three temporal layers.
+ // S0 has 5% overshoot, S1 has 25% overshoot.
+ current_input_allocation_.SetBitrate(0, 0, 180000);
+ current_input_allocation_.SetBitrate(0, 1, 60000);
+ current_input_allocation_.SetBitrate(0, 2, 60000);
+ current_input_allocation_.SetBitrate(1, 0, 400000);
+ current_input_allocation_.SetBitrate(1, 1, 150000);
+ current_input_allocation_.SetBitrate(1, 2, 150000);
+ target_framerate_fps_ = 30;
+ // Run twice, once configured as simulcast and once as VP9 SVC.
+ for (int i = 0; i < 2; ++i) {
+ SetUpAdjuster(2, 3, i == 0);
+ InsertFrames({{1.05, 1.05, 1.05}, {1.25, 1.25, 1.25}}, kWindowSizeMs);
+ current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
+ current_input_allocation_, target_framerate_fps_);
+ VideoBitrateAllocation expected_allocation;
+ for (size_t ti = 0; ti < 3; ++ti) {
+ expected_allocation.SetBitrate(
+ 0, ti,
+ static_cast<uint32_t>(current_input_allocation_.GetBitrate(0, ti) /
+ 1.05));
+ expected_allocation.SetBitrate(
+ 1, ti,
+ static_cast<uint32_t>(current_input_allocation_.GetBitrate(1, ti) /
+ 1.25));
+ }
+ ExpectNear(expected_allocation, current_adjusted_allocation_, 0.01);
+ }
+}
+
+} // namespace webrtc
diff --git a/video/encoder_overshoot_detector.cc b/video/encoder_overshoot_detector.cc
new file mode 100644
index 0000000..f1fa3bf
--- /dev/null
+++ b/video/encoder_overshoot_detector.cc
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/encoder_overshoot_detector.h"
+
+#include <algorithm>
+
+namespace webrtc {
+
+EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms)
+ : window_size_ms_(window_size_ms),
+ time_last_update_ms_(-1),
+ sum_utilization_factors_(0.0),
+ target_bitrate_(DataRate::Zero()),
+ target_framerate_fps_(0),
+ buffer_level_bits_(0) {}
+
+EncoderOvershootDetector::~EncoderOvershootDetector() = default;
+
+void EncoderOvershootDetector::SetTargetRate(DataRate target_bitrate,
+ double target_framerate_fps,
+ int64_t time_ms) {
+ // First leak bits according to the previous target rate.
+ if (target_bitrate_ != DataRate::Zero()) {
+ LeakBits(time_ms);
+ } else if (target_bitrate != DataRate::Zero()) {
+ // Stream was just enabled, reset state.
+ time_last_update_ms_ = time_ms;
+ utilization_factors_.clear();
+ sum_utilization_factors_ = 0.0;
+ buffer_level_bits_ = 0;
+ }
+
+ target_bitrate_ = target_bitrate;
+ target_framerate_fps_ = target_framerate_fps;
+}
+
+void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) {
+ // Leak bits from the virtual pacer buffer, according to the current target
+ // bitrate.
+ LeakBits(time_ms);
+
+ // Ideal size of a frame given the current rates.
+ const int64_t ideal_frame_size = IdealFrameSizeBits();
+ if (ideal_frame_size == 0) {
+ // Frame without updated bitrate and/or framerate, ignore it.
+ return;
+ }
+
+ // Add new frame to the buffer level. If doing so exceeds the ideal buffer
+ // size, penalize this frame but cap overshoot to current buffer level rather
+ // than size of this frame. This is done so that a single large frame is not
+ // penalized if the encoder afterwards compensates by dropping frames and/or
+ // reducing frame size. If however a large frame is followed by more data,
+ // we cannot pace that next frame out within one frame space.
+ const int64_t bitsum = (bytes * 8) + buffer_level_bits_;
+ int64_t overshoot_bits = 0;
+ if (bitsum > ideal_frame_size) {
+ overshoot_bits = std::min(buffer_level_bits_, bitsum - ideal_frame_size);
+ }
+
+ // Add entry for the (over) utilization for this frame. Factor is capped
+ // at 1.0 so that we don't risk overshooting on sudden changes.
+ double frame_utilization_factor;
+ if (utilization_factors_.empty()) {
+ // First frame, cannot estimate overshoot based on previous one so
+ // for this particular frame, just like as size vs optimal size.
+ frame_utilization_factor =
+ std::max(1.0, static_cast<double>(bytes) * 8 / ideal_frame_size);
+ } else {
+ frame_utilization_factor =
+ 1.0 + (static_cast<double>(overshoot_bits) / ideal_frame_size);
+ }
+ utilization_factors_.emplace_back(frame_utilization_factor, time_ms);
+ sum_utilization_factors_ += frame_utilization_factor;
+
+ // Remove the overshot bits from the virtual buffer so we don't penalize
+ // those bits multiple times.
+ buffer_level_bits_ -= overshoot_bits;
+ buffer_level_bits_ += bytes * 8;
+}
+
+absl::optional<double> EncoderOvershootDetector::GetUtilizationFactor(
+ int64_t time_ms) {
+ // Cull old data points.
+ const int64_t cutoff_time_ms = time_ms - window_size_ms_;
+ while (!utilization_factors_.empty() &&
+ utilization_factors_.front().update_time_ms < cutoff_time_ms) {
+ // Make sure sum is never allowed to become negative due rounding errors.
+ sum_utilization_factors_ =
+ std::max(0.0, sum_utilization_factors_ -
+ utilization_factors_.front().utilization_factor);
+ utilization_factors_.pop_front();
+ }
+
+ // No data points within window, return.
+ if (utilization_factors_.empty()) {
+ return absl::nullopt;
+ }
+
+ // TODO(sprang): Consider changing from arithmetic mean to some other
+ // function such as 90th percentile.
+ return sum_utilization_factors_ / utilization_factors_.size();
+}
+
+void EncoderOvershootDetector::Reset() {
+ time_last_update_ms_ = -1;
+ utilization_factors_.clear();
+ target_bitrate_ = DataRate::Zero();
+ sum_utilization_factors_ = 0.0;
+ target_framerate_fps_ = 0.0;
+ buffer_level_bits_ = 0;
+}
+
+int64_t EncoderOvershootDetector::IdealFrameSizeBits() const {
+ if (target_framerate_fps_ <= 0 || target_bitrate_ == DataRate::Zero()) {
+ return 0;
+ }
+
+ // Current ideal frame size, based on the current target bitrate.
+ return static_cast<int64_t>(
+ (target_bitrate_.bps() + target_framerate_fps_ / 2) /
+ target_framerate_fps_);
+}
+
+void EncoderOvershootDetector::LeakBits(int64_t time_ms) {
+ if (time_last_update_ms_ != -1 && target_bitrate_ > DataRate::Zero()) {
+ int64_t time_delta_ms = time_ms - time_last_update_ms_;
+ // Leak bits according to the current target bitrate.
+ int64_t leaked_bits = std::min(
+ buffer_level_bits_, (target_bitrate_.bps() * time_delta_ms) / 1000);
+ buffer_level_bits_ -= leaked_bits;
+ }
+ time_last_update_ms_ = time_ms;
+}
+
+} // namespace webrtc
diff --git a/video/encoder_overshoot_detector.h b/video/encoder_overshoot_detector.h
new file mode 100644
index 0000000..4fca2d9
--- /dev/null
+++ b/video/encoder_overshoot_detector.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_ENCODER_OVERSHOOT_DETECTOR_H_
+#define VIDEO_ENCODER_OVERSHOOT_DETECTOR_H_
+
+#include <deque>
+
+#include "absl/types/optional.h"
+#include "api/units/data_rate.h"
+
+namespace webrtc {
+
+class EncoderOvershootDetector {
+ public:
+ explicit EncoderOvershootDetector(int64_t window_size_ms);
+ ~EncoderOvershootDetector();
+
+ void SetTargetRate(DataRate target_bitrate,
+ double target_framerate_fps,
+ int64_t time_ms);
+ void OnEncodedFrame(size_t bytes, int64_t time_ms);
+ absl::optional<double> GetUtilizationFactor(int64_t time_ms);
+ void Reset();
+
+ private:
+ int64_t IdealFrameSizeBits() const;
+ void LeakBits(int64_t time_ms);
+
+ const int64_t window_size_ms_;
+ int64_t time_last_update_ms_;
+ struct BitrateUpdate {
+ BitrateUpdate(double utilization_factor, int64_t update_time_ms)
+ : utilization_factor(utilization_factor),
+ update_time_ms(update_time_ms) {}
+ double utilization_factor;
+ int64_t update_time_ms;
+ };
+ std::deque<BitrateUpdate> utilization_factors_;
+ double sum_utilization_factors_;
+ DataRate target_bitrate_;
+ double target_framerate_fps_;
+ int64_t buffer_level_bits_;
+};
+
+} // namespace webrtc
+
+#endif // VIDEO_ENCODER_OVERSHOOT_DETECTOR_H_
diff --git a/video/encoder_overshoot_detector_unittest.cc b/video/encoder_overshoot_detector_unittest.cc
new file mode 100644
index 0000000..3f393df
--- /dev/null
+++ b/video/encoder_overshoot_detector_unittest.cc
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/encoder_overshoot_detector.h"
+#include "api/units/data_rate.h"
+#include "rtc_base/fake_clock.h"
+#include "rtc_base/time_utils.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+class EncoderOvershootDetectorTest : public ::testing::Test {
+ public:
+ static constexpr int kDefaultBitrateBps = 300000;
+ static constexpr double kDefaultFrameRateFps = 15;
+ EncoderOvershootDetectorTest()
+ : detector_(kWindowSizeMs),
+ target_bitrate_(DataRate::bps(kDefaultBitrateBps)),
+ target_framerate_fps_(kDefaultFrameRateFps) {}
+
+ protected:
+ void RunConstantUtilizationTest(double actual_utilization_factor,
+ double expected_utilization_factor,
+ double allowed_error,
+ int64_t test_duration_ms) {
+ const int frame_size_bytes =
+ static_cast<int>(actual_utilization_factor *
+ (target_bitrate_.bps() / target_framerate_fps_) / 8);
+ detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
+ rtc::TimeMillis());
+
+ if (rtc::TimeMillis() == 0) {
+ // Encode a first frame which by definition has no overuse factor.
+ detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis());
+ clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec /
+ target_framerate_fps_);
+ }
+
+ int64_t runtime_us = 0;
+ while (runtime_us < test_duration_ms * 1000) {
+ detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis());
+ runtime_us += rtc::kNumMicrosecsPerSec / target_framerate_fps_;
+ clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec /
+ target_framerate_fps_);
+ }
+
+ absl::optional<double> utilization_factor =
+ detector_.GetUtilizationFactor(rtc::TimeMillis());
+ EXPECT_NEAR(utilization_factor.value_or(-1), expected_utilization_factor,
+ allowed_error);
+ }
+
+ static constexpr int64_t kWindowSizeMs = 3000;
+ EncoderOvershootDetector detector_;
+ rtc::ScopedFakeClock clock_;
+ DataRate target_bitrate_;
+ double target_framerate_fps_;
+};
+
+TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) {
+ const int frame_size_bytes = 1000;
+ const int64_t time_interval_ms = 33;
+ detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
+ rtc::TimeMillis());
+
+ // No data points, can't determine overshoot rate.
+ EXPECT_FALSE(detector_.GetUtilizationFactor(rtc::TimeMillis()).has_value());
+
+ detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis());
+ clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec * time_interval_ms);
+ EXPECT_TRUE(detector_.GetUtilizationFactor(rtc::TimeMillis()).has_value());
+}
+
+TEST_F(EncoderOvershootDetectorTest, OptimalSize) {
+ // Optimally behaved encoder.
+ // Allow some error margin due to rounding errors, eg due to frame
+ // interval not being an integer.
+ RunConstantUtilizationTest(1.0, 1.0, 0.01, kWindowSizeMs);
+}
+
+TEST_F(EncoderOvershootDetectorTest, Undershoot) {
+ // Undershoot, reported utilization factor should be capped to 1.0 so
+ // that we don't incorrectly boost encoder bitrate during movement.
+ RunConstantUtilizationTest(0.5, 1.0, 0.00, kWindowSizeMs);
+}
+
+TEST_F(EncoderOvershootDetectorTest, Overshoot) {
+ // Overshoot by 20%.
+ // Allow some error margin due to rounding errors.
+ RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs);
+}
+
+TEST_F(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) {
+ // Overshoot by 20%, but vary framerate and bitrate.
+ // Allow some error margin due to rounding errors.
+ RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs);
+ target_framerate_fps_ /= 2;
+ RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2);
+ target_bitrate_ = DataRate::bps(target_bitrate_.bps() / 2);
+ RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2);
+}
+
+TEST_F(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) {
+ // Overshoot by 10%, keep framerate and bitrate constant.
+ // Allow some error margin due to rounding errors.
+ RunConstantUtilizationTest(1.1, 1.1, 0.01, kWindowSizeMs);
+ // Change overshoot to 20%, run for half window and expect overshoot
+ // to be 15%.
+ RunConstantUtilizationTest(1.2, 1.15, 0.01, kWindowSizeMs / 2);
+ // Keep running at 20% overshoot, after window is full that should now
+ // be the reported overshoot.
+ RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2);
+}
+
+TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) {
+ const int ideal_frame_size_bytes =
+ (target_bitrate_.bps() / target_framerate_fps_) / 8;
+ detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
+ rtc::TimeMillis());
+
+ // Test scenario with average bitrate matching the target bitrate, but
+ // with some utilization factor penalty as the frames can't be paced out
+ // on the network at the target rate.
+ // Insert a series of four frames:
+ // 1) 20% overshoot, not penalized as buffer if empty.
+ // 2) 20% overshoot, the 20% overshoot from the first frame is penalized.
+ // 3) 20% undershoot, negating the overshoot from the last frame.
+ // 4) 20% undershoot, no penalty.
+ // On average then utilization penalty is thus 5%.
+
+ int64_t runtime_us = 0;
+ int i = 0;
+ while (runtime_us < kWindowSizeMs * rtc::kNumMicrosecsPerMillisec) {
+ runtime_us += rtc::kNumMicrosecsPerSec / target_framerate_fps_;
+ clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / target_framerate_fps_);
+ int frame_size_bytes = (i++ % 4 < 2) ? (ideal_frame_size_bytes * 120) / 100
+ : (ideal_frame_size_bytes * 80) / 100;
+ detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis());
+ }
+
+ absl::optional<double> utilization_factor =
+ detector_.GetUtilizationFactor(rtc::TimeMillis());
+ EXPECT_NEAR(utilization_factor.value_or(-1), 1.05, 0.01);
+}
+
+} // namespace webrtc
diff --git a/video/end_to_end_tests/bandwidth_tests.cc b/video/end_to_end_tests/bandwidth_tests.cc
index 15327a9..17bb985 100644
--- a/video/end_to_end_tests/bandwidth_tests.cc
+++ b/video/end_to_end_tests/bandwidth_tests.cc
@@ -266,6 +266,12 @@
}
TEST_F(BandwidthEndToEndTest, ReportsSetEncoderRates) {
+ // If these fields trial are on, we get lower bitrates than expected by this
+ // test, due to the packetization overhead and encoder pushback.
+ webrtc::test::ScopedFieldTrials field_trials(
+ std::string(field_trial::GetFieldTrialString()) +
+ "WebRTC-SubtractPacketizationOverhead/Disabled/"
+ "WebRTC-VideoRateControl/bitrate_adjuster:false/");
class EncoderRateStatsTest : public test::EndToEndTest,
public test::FakeEncoder {
public:
diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc
index 0585ef9..8338d84 100644
--- a/video/video_send_stream_tests.cc
+++ b/video/video_send_stream_tests.cc
@@ -2754,11 +2754,12 @@
static const int kMaxBitrateKbps = 413;
static const int kIncreasedStartBitrateKbps = 451;
static const int kIncreasedMaxBitrateKbps = 597;
- // If this field trial is on, we get lower bitrates than expected by this
- // test, due to the packetization overhead.
+ // If these fields trial are on, we get lower bitrates than expected by this
+ // test, due to the packetization overhead and encoder pushback.
webrtc::test::ScopedFieldTrials field_trials(
std::string(field_trial::GetFieldTrialString()) +
- "WebRTC-SubtractPacketizationOverhead/Disabled/");
+ "WebRTC-SubtractPacketizationOverhead/Disabled/"
+ "WebRTC-VideoRateControl/bitrate_adjuster:false/");
class EncoderBitrateThresholdObserver : public test::SendTest,
public test::FakeEncoder {
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index ea59b0a..f563f4f 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -26,6 +26,7 @@
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/quality_scaling_experiment.h"
+#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/location.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
@@ -364,6 +365,7 @@
source_proxy_(new VideoSourceProxy(this)),
sink_(nullptr),
settings_(settings),
+ rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
video_sender_(Clock::GetRealTimeClock(), this),
overuse_detector_(std::move(overuse_detector)),
encoder_stats_observer_(encoder_stats_observer),
@@ -449,7 +451,7 @@
degradation_preference_ = degradation_preference;
if (encoder_)
- ConfigureQualityScaler();
+ ConfigureQualityScaler(encoder_->GetEncoderInfo());
if (!IsFramerateScalingEnabled(degradation_preference) &&
max_framerate_ != -1) {
@@ -645,6 +647,12 @@
field_trial::IsDisabled(kFrameDropperFieldTrial) ||
(num_layers > 1 && codec.mode == VideoCodecMode::kScreensharing);
+ VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
+ if (rate_control_settings_.UseEncoderBitrateAdjuster()) {
+ bitrate_adjuster_ = absl::make_unique<EncoderBitrateAdjuster>(codec);
+ bitrate_adjuster_->OnEncoderInfo(info);
+ }
+
if (rate_allocator_ && last_observed_bitrate_bps_ > 0) {
// We have a new rate allocator instance and already configured target
// bitrate. Update the rate allocation and notify observsers.
@@ -672,12 +680,13 @@
max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps);
overuse_detector_->OnTargetFramerateUpdated(target_framerate);
- ConfigureQualityScaler();
+ ConfigureQualityScaler(info);
}
-void VideoStreamEncoder::ConfigureQualityScaler() {
+void VideoStreamEncoder::ConfigureQualityScaler(
+ const VideoEncoder::EncoderInfo& encoder_info) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- const auto scaling_settings = encoder_->GetEncoderInfo().scaling_settings;
+ const auto scaling_settings = encoder_info.scaling_settings;
const bool quality_scaling_allowed =
IsResolutionScalingEnabled(degradation_preference_) &&
scaling_settings.thresholds;
@@ -840,6 +849,10 @@
bitrate_observer_->OnBitrateAllocationUpdated(bitrate_allocation);
}
+ if (bitrate_adjuster_) {
+ return bitrate_adjuster_->AdjustRateAllocation(bitrate_allocation,
+ framerate_fps);
+ }
return bitrate_allocation;
}
@@ -987,7 +1000,21 @@
if (info.implementation_name != encoder_info_.implementation_name) {
encoder_stats_observer_->OnEncoderImplementationChanged(
info.implementation_name);
+ if (bitrate_adjuster_) {
+ // Encoder implementation changed, reset overshoot detector states.
+ bitrate_adjuster_->Reset();
+ }
}
+
+ if (bitrate_adjuster_) {
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ if (info.fps_allocation[si] != encoder_info_.fps_allocation[si]) {
+ bitrate_adjuster_->OnEncoderInfo(info);
+ break;
+ }
+ }
+ }
+
encoder_info_ = info;
input_framerate_.Update(1u, clock_->TimeInMilliseconds());
@@ -1018,26 +1045,25 @@
sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
int64_t time_sent_us = rtc::TimeMicros();
- uint32_t timestamp = encoded_image.Timestamp();
- const int qp = encoded_image.qp_;
- int64_t capture_time_us =
- encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
+ // We are only interested in propagating the meta-data about the image, not
+ // encoded data itself, to the post encode function. Since we cannot be sure
+ // the pointer will still be valid when run on the task queue, set it to null.
+ EncodedImage encoded_image_metadata = encoded_image;
+ encoded_image_metadata.set_buffer(nullptr, 0);
- absl::optional<int> encode_duration_us;
- if (encoded_image.timing_.flags != VideoSendTiming::kInvalid) {
- encode_duration_us.emplace(
- // TODO(nisse): Maybe use capture_time_ms_ rather than encode_start_ms_?
- rtc::kNumMicrosecsPerMillisec *
- (encoded_image.timing_.encode_finish_ms -
- encoded_image.timing_.encode_start_ms));
+ int temporal_index = 0;
+ if (codec_specific_info) {
+ if (codec_specific_info->codecType == kVideoCodecVP9) {
+ temporal_index = codec_specific_info->codecSpecific.VP9.temporal_idx;
+ } else if (codec_specific_info->codecType == kVideoCodecVP8) {
+ temporal_index = codec_specific_info->codecSpecific.VP8.temporalIdx;
+ }
+ }
+ if (temporal_index == kNoTemporalIdx) {
+ temporal_index = 0;
}
- // Run post encode tasks, such as overuse detection and frame rate/drop
- // stats for internal encoders.
- const size_t frame_size = encoded_image.size();
- const bool keyframe = encoded_image._frameType == FrameType::kVideoFrameKey;
- RunPostEncode(timestamp, time_sent_us, capture_time_us, encode_duration_us,
- qp, frame_size, keyframe);
+ RunPostEncode(encoded_image_metadata, time_sent_us, temporal_index);
if (result.error == Result::OK) {
// In case of an internal encoder running on a separate thread, the
@@ -1372,26 +1398,35 @@
return adapt_counters_[degradation_preference_];
}
-void VideoStreamEncoder::RunPostEncode(uint32_t frame_timestamp,
+void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image,
int64_t time_sent_us,
- int64_t capture_time_us,
- absl::optional<int> encode_durations_us,
- int qp,
- size_t frame_size_bytes,
- bool keyframe) {
+ int temporal_index) {
if (!encoder_queue_.IsCurrent()) {
- encoder_queue_.PostTask([this, frame_timestamp, time_sent_us, qp,
- capture_time_us, encode_durations_us,
- frame_size_bytes, keyframe] {
- RunPostEncode(frame_timestamp, time_sent_us, capture_time_us,
- encode_durations_us, qp, frame_size_bytes, keyframe);
- });
+ encoder_queue_.PostTask(
+ [this, encoded_image, time_sent_us, temporal_index] {
+ RunPostEncode(encoded_image, time_sent_us, temporal_index);
+ });
return;
}
RTC_DCHECK_RUN_ON(&encoder_queue_);
- if (frame_size_bytes > 0) {
- frame_dropper_.Fill(frame_size_bytes, !keyframe);
+
+ absl::optional<int> encode_duration_us;
+ if (encoded_image.timing_.flags != VideoSendTiming::kInvalid) {
+ encode_duration_us =
+ // TODO(nisse): Maybe use capture_time_ms_ rather than encode_start_ms_?
+ rtc::kNumMicrosecsPerMillisec *
+ (encoded_image.timing_.encode_finish_ms -
+ encoded_image.timing_.encode_start_ms);
+ }
+
+ // Run post encode tasks, such as overuse detection and frame rate/drop
+ // stats for internal encoders.
+ const size_t frame_size = encoded_image.size();
+ const bool keyframe = encoded_image._frameType == FrameType::kVideoFrameKey;
+
+ if (frame_size > 0) {
+ frame_dropper_.Fill(frame_size, !keyframe);
}
if (encoder_info_.has_internal_source) {
@@ -1404,10 +1439,15 @@
}
}
- overuse_detector_->FrameSent(frame_timestamp, time_sent_us, capture_time_us,
- encode_durations_us);
- if (quality_scaler_ && qp >= 0)
- quality_scaler_->ReportQp(qp);
+ overuse_detector_->FrameSent(
+ encoded_image.Timestamp(), time_sent_us,
+ encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec,
+ encode_duration_us);
+ if (quality_scaler_ && encoded_image.qp_ >= 0)
+ quality_scaler_->ReportQp(encoded_image.qp_);
+ if (bitrate_adjuster_) {
+ bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index);
+ }
}
// Class holding adaptation information.
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 577ca5b..feed55b 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -29,9 +29,11 @@
#include "modules/video_coding/video_coding_impl.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/event.h"
+#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/rate_statistics.h"
#include "rtc_base/sequenced_task_checker.h"
#include "rtc_base/task_queue.h"
+#include "video/encoder_bitrate_adjuster.h"
#include "video/overuse_frame_detector.h"
namespace webrtc {
@@ -106,7 +108,7 @@
size_t max_data_payload_length);
void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_);
- void ConfigureQualityScaler();
+ void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
// Implements VideoSinkInterface.
void OnFrame(const VideoFrame& video_frame) override;
@@ -180,13 +182,9 @@
void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(&encoder_queue_);
VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
AdaptReason reason) RTC_RUN_ON(&encoder_queue_);
- void RunPostEncode(uint32_t frame_timestamp,
+ void RunPostEncode(EncodedImage encoded_image,
int64_t time_sent_us,
- int64_t capture_time_us,
- absl::optional<int> encode_durations_us,
- int qp,
- size_t frame_size_bytes,
- bool keyframe);
+ int temporal_index);
rtc::Event shutdown_event_;
@@ -201,6 +199,7 @@
const std::unique_ptr<VideoSourceProxy> source_proxy_;
EncoderSink* sink_;
const VideoStreamEncoderSettings settings_;
+ const RateControlSettings rate_control_settings_;
vcm::VideoSender video_sender_ RTC_GUARDED_BY(&encoder_queue_);
const std::unique_ptr<OveruseFrameDetector> overuse_detector_
@@ -295,6 +294,9 @@
// the worker thread.
std::atomic<int> pending_frame_drops_;
+ std::unique_ptr<EncoderBitrateAdjuster> bitrate_adjuster_
+ RTC_GUARDED_BY(&encoder_queue_);
+
// All public methods are proxied to |encoder_queue_|. It must must be
// destroyed first to make sure no tasks are run that use other members.
rtc::TaskQueue encoder_queue_;
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index fba99fe..68a30a1 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -25,6 +25,7 @@
#include "rtc_base/fake_clock.h"
#include "rtc_base/logging.h"
#include "rtc_base/ref_counted_object.h"
+#include "system_wrappers/include/field_trial.h"
#include "system_wrappers/include/metrics.h"
#include "system_wrappers/include/sleep.h"
#include "test/encoder_settings.h"
@@ -3300,7 +3301,13 @@
// Make encoder produce frames at double the expected bitrate during 3 seconds
// of video, verify number of drops. Rate needs to be slightly changed in
// order to force the rate to be reconfigured.
- fake_encoder_.SimulateOvershoot(2.0);
+ double overshoot_factor = 2.0;
+ if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
+ // With bitrate adjuster, when need to overshoot even more to trigger
+ // frame dropping.
+ overshoot_factor *= 2;
+ }
+ fake_encoder_.SimulateOvershoot(overshoot_factor);
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps + 1000, 0, 0);
num_dropped = 0;
for (int i = 0; i < kNumFramesInRun; ++i) {
