VideoStreamEncoder report VideoLayersAllocation for simulcast
Adds support for Vp8 simulcast.
Bug: webrtc:12000
Change-Id: Ib24fd0542642b023ec35f7a7bdc4880d72365edf
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/187341
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32416}
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index d75ec8f..b024548 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -21,7 +21,6 @@
"hdr_metadata.h",
"video_content_type.cc",
"video_content_type.h",
- "video_layers_allocation.h",
"video_rotation.h",
"video_timing.cc",
"video_timing.h",
@@ -182,6 +181,13 @@
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
+rtc_source_set("video_layers_allocation") {
+ visibility = [ "*" ]
+ sources = [ "video_layers_allocation.h" ]
+ deps = [ "../units:data_rate" ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ]
+}
+
rtc_library("video_bitrate_allocator") {
visibility = [ "*" ]
sources = [
@@ -264,6 +270,7 @@
":video_bitrate_allocator_factory",
":video_codec_constants",
":video_frame",
+ ":video_layers_allocation",
"..:rtp_parameters",
"..:scoped_refptr",
"../:fec_controller_api",
diff --git a/api/video/video_stream_encoder_interface.h b/api/video/video_stream_encoder_interface.h
index ed9989c..34fa642 100644
--- a/api/video/video_stream_encoder_interface.h
+++ b/api/video/video_stream_encoder_interface.h
@@ -19,6 +19,7 @@
#include "api/scoped_refptr.h"
#include "api/units/data_rate.h"
#include "api/video/video_bitrate_allocator.h"
+#include "api/video/video_layers_allocation.h"
#include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h"
#include "api/video_codecs/video_encoder.h"
@@ -52,6 +53,9 @@
virtual void OnBitrateAllocationUpdated(
const VideoBitrateAllocation& allocation) = 0;
+
+ virtual void OnVideoLayersAllocationUpdated(
+ VideoLayersAllocation allocation) = 0;
};
// If the resource is overusing, the VideoStreamEncoder will try to reduce
diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn
index 9a9752d..9761790 100644
--- a/modules/rtp_rtcp/BUILD.gn
+++ b/modules/rtp_rtcp/BUILD.gn
@@ -110,6 +110,7 @@
"../../api/transport/rtp:dependency_descriptor",
"../../api/units:time_delta",
"../../api/video:video_frame",
+ "../../api/video:video_layers_allocation",
"../../api/video:video_rtp_headers",
"../../common_video",
"../../rtc_base:checks",
@@ -278,6 +279,7 @@
"../../api/video:video_codec_constants",
"../../api/video:video_frame",
"../../api/video:video_frame_type",
+ "../../api/video:video_layers_allocation",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../call:rtp_interfaces",
@@ -537,6 +539,7 @@
"../../api/video:video_bitrate_allocator",
"../../api/video:video_codec_constants",
"../../api/video:video_frame",
+ "../../api/video:video_layers_allocation",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../call:rtp_receiver",
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 115c71a..c6774dc 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -226,6 +226,7 @@
"../api/video:video_bitrate_allocator_factory",
"../api/video:video_codec_constants",
"../api/video:video_frame",
+ "../api/video:video_layers_allocation",
"../api/video:video_rtp_headers",
"../api/video:video_stream_encoder",
"../api/video_codecs:video_codecs_api",
diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc
index 82f8aa8..4673811 100644
--- a/video/video_send_stream_impl.cc
+++ b/video/video_send_stream_impl.cc
@@ -472,6 +472,20 @@
}
}
+void VideoSendStreamImpl::OnVideoLayersAllocationUpdated(
+ VideoLayersAllocation allocation) {
+ if (!worker_queue_->IsCurrent()) {
+ auto ptr = weak_ptr_;
+ worker_queue_->PostTask([allocation = std::move(allocation), ptr] {
+ if (!ptr.get())
+ return;
+ ptr->OnVideoLayersAllocationUpdated(allocation);
+ });
+ return;
+ }
+ // TODO(bugs.webrtc.org/12000): Implement
+}
+
void VideoSendStreamImpl::SignalEncoderActive() {
RTC_DCHECK_RUN_ON(worker_queue_);
if (rtp_video_sender_->IsActive()) {
diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h
index d3f5058..41a7859 100644
--- a/video/video_send_stream_impl.h
+++ b/video/video_send_stream_impl.h
@@ -121,6 +121,8 @@
void OnBitrateAllocationUpdated(
const VideoBitrateAllocation& allocation) override;
+ void OnVideoLayersAllocationUpdated(
+ VideoLayersAllocation allocation) override;
// Implements EncodedImageCallback. The implementation routes encoded frames
// to the |payload_router_| and |config.pre_encode_callback| if set.
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index f4283bd..d261ae0 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -198,6 +198,73 @@
return new_allocation;
}
+// Converts a VideoBitrateAllocation that contains allocated bitrate per layer,
+// and an EncoderInfo that contains information about the actual encoder
+// structure used by a codec. Stream structures can be Ksvc, Full SVC, Simulcast
+// etc.
+VideoLayersAllocation CreateVideoLayersAllocation(
+ const VideoCodec& encoder_config,
+ const VideoEncoder::RateControlParameters& current_rate,
+ const VideoEncoder::EncoderInfo& encoder_info) {
+ const VideoBitrateAllocation& target_bitrate = current_rate.target_bitrate;
+ VideoLayersAllocation layers_allocation;
+ if (target_bitrate.get_sum_bps() == 0) {
+ return layers_allocation;
+ }
+
+ if (encoder_config.numberOfSimulcastStreams > 0) {
+ layers_allocation.resolution_and_frame_rate_is_valid = true;
+ for (int si = 0; si < encoder_config.numberOfSimulcastStreams; ++si) {
+ if (!target_bitrate.IsSpatialLayerUsed(si) ||
+ target_bitrate.GetSpatialLayerSum(si) == 0) {
+ break;
+ }
+ layers_allocation.active_spatial_layers.emplace_back();
+ VideoLayersAllocation::SpatialLayer& spatial_layer =
+ layers_allocation.active_spatial_layers.back();
+ spatial_layer.width = encoder_config.simulcastStream[si].width;
+ spatial_layer.height = encoder_config.simulcastStream[si].height;
+ spatial_layer.rtp_stream_index = si;
+ spatial_layer.spatial_id = 0;
+ auto frame_rate_fraction =
+ VideoEncoder::EncoderInfo::kMaxFramerateFraction;
+ if (encoder_info.fps_allocation[si].size() == 1) {
+ // One TL is signalled to be used by the encoder. Do not distribute
+ // bitrate allocation across TLs (use sum at tl:0).
+ spatial_layer.target_bitrate_per_temporal_layer.push_back(
+ DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si)));
+ frame_rate_fraction = encoder_info.fps_allocation[si][0];
+ } else { // Temporal layers are supported.
+ uint32_t temporal_layer_bitrate_bps = 0;
+ for (size_t ti = 0;
+ ti < encoder_config.simulcastStream[si].numberOfTemporalLayers;
+ ++ti) {
+ if (!target_bitrate.HasBitrate(si, ti)) {
+ break;
+ }
+ if (ti < encoder_info.fps_allocation[si].size()) {
+ // Use frame rate of the top used temporal layer.
+ frame_rate_fraction = encoder_info.fps_allocation[si][ti];
+ }
+ temporal_layer_bitrate_bps += target_bitrate.GetBitrate(si, ti);
+ spatial_layer.target_bitrate_per_temporal_layer.push_back(
+ DataRate::BitsPerSec(temporal_layer_bitrate_bps));
+ }
+ }
+ // Encoder may drop frames internally if `maxFramerate` is set.
+ spatial_layer.frame_rate_fps = std::min(
+ static_cast<uint8_t>(encoder_config.simulcastStream[si].maxFramerate),
+ static_cast<uint8_t>(
+ (current_rate.framerate_fps * frame_rate_fraction) /
+ VideoEncoder::EncoderInfo::kMaxFramerateFraction));
+ }
+ } else {
+ // TODO(bugs.webrtc.org/12000): Implement support for kSVC and full SVC.
+ }
+
+ return layers_allocation;
+}
+
} // namespace
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
@@ -1124,6 +1191,12 @@
rate_settings.rate_control.bitrate,
static_cast<uint32_t>(rate_settings.rate_control.framerate_fps + 0.5));
stream_resource_manager_.SetEncoderRates(rate_settings.rate_control);
+ if (settings_.allocation_cb_type ==
+ VideoStreamEncoderSettings::BitrateAllocationCallbackType::
+ kVideoLayersAllocation) {
+ sink_->OnVideoLayersAllocationUpdated(CreateVideoLayersAllocation(
+ send_codec_, rate_settings.rate_control, encoder_->GetEncoderInfo()));
+ }
}
if ((settings_.allocation_cb_type ==
VideoStreamEncoderSettings::BitrateAllocationCallbackType::
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index b8c69c6..0d34c01 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -1254,6 +1254,16 @@
return number_of_bitrate_allocations_;
}
+ VideoLayersAllocation GetLastVideoLayersAllocation() {
+ MutexLock lock(&mutex_);
+ return last_layers_allocation_;
+ }
+
+ int number_of_layers_allocations() const {
+ MutexLock lock(&mutex_);
+ return number_of_layers_allocations_;
+ }
+
private:
Result OnEncodedImage(
const EncodedImage& encoded_image,
@@ -1296,6 +1306,24 @@
last_bitrate_allocation_ = allocation;
}
+ void OnVideoLayersAllocationUpdated(
+ VideoLayersAllocation allocation) override {
+ MutexLock lock(&mutex_);
+ ++number_of_layers_allocations_;
+ last_layers_allocation_ = allocation;
+ rtc::StringBuilder log;
+ for (const auto& layer : allocation.active_spatial_layers) {
+ log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
+ << "[";
+ for (const auto target_bitrate :
+ layer.target_bitrate_per_temporal_layer) {
+ log << target_bitrate.kbps() << ",";
+ }
+ log << "]";
+ }
+ RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
+ }
+
TimeController* const time_controller_;
mutable Mutex mutex_;
TestEncoder* test_encoder_;
@@ -1313,6 +1341,8 @@
int min_transmit_bitrate_bps_ = 0;
VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
+ VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
+ int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
};
class VideoBitrateAllocatorProxyFactory
@@ -4013,6 +4043,100 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForV8Simulcast) {
+ ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
+ VideoStreamEncoderSettings::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ const int kDefaultFps = 30;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+ // kLowTargetBitrateBps is only enough for one spatial layer.
+ ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
+
+ VideoBitrateAllocation bitrate_allocation =
+ fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
+ // Check that encoder has been updated too, not just allocation observer.
+ EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
+ AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
+
+ // VideoLayersAllocation might be updated if frame rate change.
+ int number_of_layers_allocation = 1;
+ const int64_t start_time_ms = CurrentTimeMs();
+ while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
+ if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
+ number_of_layers_allocation = sink_.number_of_layers_allocations();
+ VideoLayersAllocation new_allocation =
+ sink_.GetLastVideoLayersAllocation();
+ ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
+ EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
+ last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
+ EXPECT_EQ(new_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer);
+ last_layer_allocation = new_allocation;
+ }
+ }
+ EXPECT_LE(sink_.number_of_layers_allocations(), 3);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
+ ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
+ VideoStreamEncoderSettings::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+ // kLowTargetBitrateBps is only enough for one spatial layer.
+ ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer[0],
+ DataRate::BitsPerSec(kLowTargetBitrateBps));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
+ DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
+ DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
+ last_layer_allocation = sink_.GetLastVideoLayersAllocation();
+ ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
+ EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer[0],
+ DataRate::Zero());
+
+ video_stream_encoder_->Stop();
+}
+
TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
// 2 TLs configured, temporal layers supported by encoder.
const int kNumTemporalLayers = 2;
