Fix for potential RTCP XR target bitrate issue.
Fix issue where RTCP XR target bitrate could be incorrect when temporal layers are configured but it is not supported by encoder.
Bug: webrtc:10475
Change-Id: Ib525eb5f0ad8392e88d2579930ac8e459a1d194b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/128778
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27268}diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 8b32864..fcae1d6 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -185,6 +185,31 @@
}
return experiment_groups;
}
+
+// Limit allocation across TLs in bitrate allocation according to number of TLs
+// in EncoderInfo.
+VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
+ const VideoBitrateAllocation& allocation,
+ const VideoEncoder::EncoderInfo& encoder_info) {
+ if (allocation.get_sum_bps() == 0) {
+ return allocation;
+ }
+ VideoBitrateAllocation new_allocation;
+ for (int si = 0; si < kMaxSpatialLayers; ++si) {
+ if (encoder_info.fps_allocation[si].size() == 1 &&
+ allocation.IsSpatialLayerUsed(si)) {
+ // One TL is signalled to be used by the encoder. Do not distribute
+ // bitrate allocation across TLs (use sum at ti:0).
+ new_allocation.SetBitrate(si, 0, allocation.GetSpatialLayerSum(si));
+ } else {
+ for (int ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ if (allocation.HasBitrate(si, ti))
+ new_allocation.SetBitrate(si, ti, allocation.GetBitrate(si, ti));
+ }
+ }
+ }
+ return new_allocation;
+}
} // namespace
// VideoSourceProxy is responsible ensuring thread safety between calls to
@@ -722,6 +747,7 @@
}
frame_encoder_timer_.Reset();
+ last_encode_info_ms_ = absl::nullopt;
}
if (success) {
@@ -977,7 +1003,20 @@
}
if (bitrate_observer_ && bitrate_allocation.get_sum_bps() > 0) {
- bitrate_observer_->OnBitrateAllocationUpdated(bitrate_allocation);
+ if (encoder_ && encoder_initialized_) {
+ // Avoid too old encoder_info_.
+ const int64_t kMaxDiffMs = 100;
+ const bool updated_recently =
+ (last_encode_info_ms_ && ((clock_->TimeInMilliseconds() -
+ *last_encode_info_ms_) < kMaxDiffMs));
+ // Update allocation according to info from encoder.
+ bitrate_observer_->OnBitrateAllocationUpdated(
+ UpdateAllocationFromEncoderInfo(
+ bitrate_allocation,
+ updated_recently ? encoder_info_ : encoder_->GetEncoderInfo()));
+ } else {
+ bitrate_observer_->OnBitrateAllocationUpdated(bitrate_allocation);
+ }
}
if (bitrate_adjuster_) {
@@ -1238,6 +1277,7 @@
}
encoder_info_ = info;
+ last_encode_info_ms_ = clock_->TimeInMilliseconds();
RTC_DCHECK_EQ(send_codec_.width, out_frame.width());
RTC_DCHECK_EQ(send_codec_.height, out_frame.height());
const VideoFrameBuffer::Type buffer_type =
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 663a020..41e865c 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -290,6 +290,7 @@
RTC_GUARDED_BY(&encoder_queue_);
absl::optional<int64_t> last_parameters_update_ms_
RTC_GUARDED_BY(&encoder_queue_);
+ absl::optional<int64_t> last_encode_info_ms_ RTC_GUARDED_BY(&encoder_queue_);
VideoEncoder::EncoderInfo encoder_info_ RTC_GUARDED_BY(&encoder_queue_);
VideoEncoderFactory::CodecInfo codec_info_ RTC_GUARDED_BY(&encoder_queue_);
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 5193af5..278e566 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -23,6 +23,7 @@
#include "media/base/video_adapter.h"
#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include "rtc_base/fake_clock.h"
#include "rtc_base/logging.h"
#include "rtc_base/ref_counted_object.h"
@@ -407,6 +408,20 @@
return frame;
}
+ void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
+ MockBitrateObserver bitrate_observer;
+ video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
+
+ EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
+ .Times(1);
+ video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTargetBitrateBps),
+ DataRate::Zero(), 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, codec_width_, codec_height_));
+ WaitForEncodedFrame(1);
+ }
+
void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
@@ -555,6 +570,12 @@
VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
}
info.is_hardware_accelerated = is_hardware_accelerated_;
+ for (int i = 0; i < kMaxSpatialLayers; ++i) {
+ if (temporal_layers_supported_[i]) {
+ int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
+ info.fps_allocation[i].resize(num_layers);
+ }
+ }
}
return info;
}
@@ -585,6 +606,12 @@
is_hardware_accelerated_ = is_hardware_accelerated;
}
+ void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
+ RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
+ rtc::CritScope lock(&local_crit_sect_);
+ temporal_layers_supported_[spatial_idx] = supported;
+ }
+
void ForceInitEncodeFailure(bool force_failure) {
rtc::CritScope lock(&local_crit_sect_);
force_init_encode_failed_ = force_failure;
@@ -735,6 +762,9 @@
bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
RTC_GUARDED_BY(local_crit_sect_);
+ absl::optional<bool>
+ temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
+ local_crit_sect_);
bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
@@ -2326,6 +2356,65 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
+ // 2 TLs configured, temporal layers supported by encoder.
+ const int kNumTemporalLayers = 2;
+ ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
+ fake_encoder_.SetTemporalLayersSupported(0, true);
+
+ // Bitrate allocated across temporal layers.
+ const int kTl0Bps = kTargetBitrateBps *
+ webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
+ kNumTemporalLayers, /*temporal_id*/ 0);
+ const int kTl1Bps = kTargetBitrateBps *
+ webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
+ kNumTemporalLayers, /*temporal_id*/ 1);
+ VideoBitrateAllocation expected_bitrate;
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
+
+ VerifyAllocatedBitrate(expected_bitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
+ // 2 TLs configured, temporal layers not supported by encoder.
+ ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
+ fake_encoder_.SetTemporalLayersSupported(0, false);
+
+ // Temporal layers not supported by the encoder.
+ // Total bitrate should be at ti:0.
+ VideoBitrateAllocation expected_bitrate;
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
+
+ VerifyAllocatedBitrate(expected_bitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
+ // 2 TLs configured, temporal layers only supported for first stream.
+ ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
+ fake_encoder_.SetTemporalLayersSupported(0, true);
+ fake_encoder_.SetTemporalLayersSupported(1, false);
+
+ const int kS0Bps = 150000;
+ const int kS0Tl0Bps =
+ kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
+ /*num_layers*/ 2, /*temporal_id*/ 0);
+ const int kS0Tl1Bps =
+ kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
+ /*num_layers*/ 2, /*temporal_id*/ 1);
+ const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
+ // Temporal layers not supported by si:1.
+ VideoBitrateAllocation expected_bitrate;
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
+ expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
+
+ VerifyAllocatedBitrate(expected_bitrate);
+ video_stream_encoder_->Stop();
+}
+
TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
const int kFrameWidth = 1280;
const int kFrameHeight = 720;
