Allow setting different number of temporal layers per simulcast layer.
Setting different number of temporal layers is supported by SimulcastEncodeAdapter and LibvpxVp8Encoder will fallback to SimulcastEncoderAdapter if InitEncode fails.
Bug: none
Change-Id: I8a09ee1e6c70a0006317957c0802d019a0d28ca2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228642
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34785}
diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h
index 71ae984..2664ae7 100644
--- a/api/rtp_parameters.h
+++ b/api/rtp_parameters.h
@@ -489,8 +489,6 @@
// Specifies the number of temporal layers for video (if the feature is
// supported by the codec implementation).
- // TODO(asapersson): Different number of temporal layers are not supported
- // per simulcast layer.
// Screencast support is experimental.
absl::optional<int> num_temporal_layers;
diff --git a/media/base/media_engine.cc b/media/base/media_engine.cc
index 36a9694..21c3787 100644
--- a/media/base/media_engine.cc
+++ b/media/base/media_engine.cc
@@ -106,15 +106,6 @@
"num_temporal_layers to an invalid number.");
}
}
- if (i > 0 && (rtp_parameters.encodings[i].num_temporal_layers !=
- rtp_parameters.encodings[i - 1].num_temporal_layers)) {
- LOG_AND_RETURN_ERROR(
- RTCErrorType::INVALID_MODIFICATION,
- "Attempted to set RtpParameters num_temporal_layers "
- "at encoding layer i: " +
- rtc::ToString(i) +
- " to a different value than other encoding layers.");
- }
}
return webrtc::RTCError::OK();
diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc
index 7c1bf6e..025a553 100644
--- a/media/engine/webrtc_video_engine_unittest.cc
+++ b/media/engine/webrtc_video_engine_unittest.cc
@@ -7706,28 +7706,6 @@
channel_->SetRtpSendParameters(last_ssrc_, parameters).type());
}
-TEST_F(WebRtcVideoChannelTest,
- SetRtpSendParametersNumTemporalLayersFailsForInvalidModification) {
- const size_t kNumSimulcastStreams = 3;
- SetUpSimulcast(true, false);
-
- // Get and set the rtp encoding parameters.
- webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_);
- EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size());
-
- // No/all layers should be set.
- parameters.encodings[0].num_temporal_layers = 1;
- EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION,
- channel_->SetRtpSendParameters(last_ssrc_, parameters).type());
-
- // Different values not supported.
- parameters.encodings[0].num_temporal_layers = 1;
- parameters.encodings[1].num_temporal_layers = 2;
- parameters.encodings[2].num_temporal_layers = 2;
- EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION,
- channel_->SetRtpSendParameters(last_ssrc_, parameters).type());
-}
-
TEST_F(WebRtcVideoChannelTest, GetAndSetRtpSendParametersNumTemporalLayers) {
const size_t kNumSimulcastStreams = 3;
SetUpSimulcast(true, false);
@@ -7767,9 +7745,9 @@
// Change the value and set it on the VideoChannel.
webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size());
- parameters.encodings[0].num_temporal_layers = 2;
+ parameters.encodings[0].num_temporal_layers = 3;
parameters.encodings[1].num_temporal_layers = 2;
- parameters.encodings[2].num_temporal_layers = 2;
+ parameters.encodings[2].num_temporal_layers = 1;
EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok());
// Verify that the new value is propagated down to the encoder.
@@ -7778,16 +7756,16 @@
webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy();
EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams);
EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size());
- EXPECT_EQ(2UL, encoder_config.simulcast_layers[0].num_temporal_layers);
+ EXPECT_EQ(3UL, encoder_config.simulcast_layers[0].num_temporal_layers);
EXPECT_EQ(2UL, encoder_config.simulcast_layers[1].num_temporal_layers);
- EXPECT_EQ(2UL, encoder_config.simulcast_layers[2].num_temporal_layers);
+ EXPECT_EQ(1UL, encoder_config.simulcast_layers[2].num_temporal_layers);
// FakeVideoSendStream calls CreateEncoderStreams, test that the vector of
// VideoStreams are created appropriately for the simulcast case.
EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size());
- EXPECT_EQ(2UL, stream->GetVideoStreams()[0].num_temporal_layers);
+ EXPECT_EQ(3UL, stream->GetVideoStreams()[0].num_temporal_layers);
EXPECT_EQ(2UL, stream->GetVideoStreams()[1].num_temporal_layers);
- EXPECT_EQ(2UL, stream->GetVideoStreams()[2].num_temporal_layers);
+ EXPECT_EQ(1UL, stream->GetVideoStreams()[2].num_temporal_layers);
// No parameter changed, encoder should not be reconfigured.
EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok());
@@ -7809,29 +7787,28 @@
channel_->SetSend(true);
frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame());
- // Change rtp encoding parameters, num_temporal_layers not changed.
+ // Change rtp encoding parameters.
webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size());
- parameters.encodings[0].min_bitrate_bps = 33000;
+ parameters.encodings[0].num_temporal_layers = 2;
+ parameters.encodings[2].num_temporal_layers = 1;
EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok());
// Verify that no value is propagated down to the encoder.
webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy();
EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams);
EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size());
- EXPECT_FALSE(encoder_config.simulcast_layers[0].num_temporal_layers);
+ EXPECT_EQ(2UL, encoder_config.simulcast_layers[0].num_temporal_layers);
EXPECT_FALSE(encoder_config.simulcast_layers[1].num_temporal_layers);
- EXPECT_FALSE(encoder_config.simulcast_layers[2].num_temporal_layers);
+ EXPECT_EQ(1UL, encoder_config.simulcast_layers[2].num_temporal_layers);
// FakeVideoSendStream calls CreateEncoderStreams, test that the vector of
// VideoStreams are created appropriately for the simulcast case.
EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size());
- EXPECT_EQ(kDefaultNumTemporalLayers,
- stream->GetVideoStreams()[0].num_temporal_layers);
+ EXPECT_EQ(2UL, stream->GetVideoStreams()[0].num_temporal_layers);
EXPECT_EQ(kDefaultNumTemporalLayers,
stream->GetVideoStreams()[1].num_temporal_layers);
- EXPECT_EQ(kDefaultNumTemporalLayers,
- stream->GetVideoStreams()[2].num_temporal_layers);
+ EXPECT_EQ(1UL, stream->GetVideoStreams()[2].num_temporal_layers);
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
diff --git a/pc/rtp_sender_receiver_unittest.cc b/pc/rtp_sender_receiver_unittest.cc
index a8140e8..f361833 100644
--- a/pc/rtp_sender_receiver_unittest.cc
+++ b/pc/rtp_sender_receiver_unittest.cc
@@ -1260,6 +1260,30 @@
DestroyVideoRtpSender();
}
+TEST_F(RtpSenderReceiverTest, VideoSenderCanSetNumTemporalLayers) {
+ CreateVideoRtpSender();
+
+ RtpParameters params = video_rtp_sender_->GetParameters();
+ params.encodings[0].num_temporal_layers = 2;
+
+ EXPECT_TRUE(video_rtp_sender_->SetParameters(params).ok());
+ params = video_rtp_sender_->GetParameters();
+ EXPECT_EQ(2, params.encodings[0].num_temporal_layers);
+
+ DestroyVideoRtpSender();
+}
+
+TEST_F(RtpSenderReceiverTest, VideoSenderDetectInvalidNumTemporalLayers) {
+ CreateVideoRtpSender();
+
+ RtpParameters params = video_rtp_sender_->GetParameters();
+ params.encodings[0].num_temporal_layers = webrtc::kMaxTemporalStreams + 1;
+ RTCError result = video_rtp_sender_->SetParameters(params);
+ EXPECT_EQ(RTCErrorType::INVALID_RANGE, result.type());
+
+ DestroyVideoRtpSender();
+}
+
TEST_F(RtpSenderReceiverTest, VideoSenderCanSetMaxFramerate) {
CreateVideoRtpSender();
diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc
index 8cb4313..6d6ab05 100644
--- a/video/video_send_stream_tests.cc
+++ b/video/video_send_stream_tests.cc
@@ -25,13 +25,18 @@
#include "call/rtp_transport_controller_send.h"
#include "call/simulated_network.h"
#include "call/video_send_stream.h"
+#include "media/engine/internal_encoder_factory.h"
+#include "media/engine/simulcast_encoder_adapter.h"
+#include "media/engine/webrtc_video_engine.h"
#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
+#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
#include "modules/rtp_rtcp/source/rtcp_sender.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
#include "modules/rtp_rtcp/source/rtp_util.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h"
+#include "modules/video_coding/codecs/interface/common_constants.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "rtc_base/checks.h"
@@ -122,6 +127,10 @@
uint8_t num_spatial_layers);
void TestRequestSourceRotateVideo(bool support_orientation_ext);
+
+ void TestTemporalLayers(VideoEncoderFactory* encoder_factory,
+ const std::string& payload_name,
+ const std::vector<int>& num_temporal_layers);
};
TEST_F(VideoSendStreamTest, CanStartStartedStream) {
@@ -3971,4 +3980,204 @@
RunBaseTest(&test);
}
+void VideoSendStreamTest::TestTemporalLayers(
+ VideoEncoderFactory* encoder_factory,
+ const std::string& payload_name,
+ const std::vector<int>& num_temporal_layers) {
+ static constexpr int kMaxBitrateBps = 1000000;
+ static constexpr int kMinFramesToObservePerStream = 8;
+
+ class TemporalLayerObserver
+ : public test::EndToEndTest,
+ public test::FrameGeneratorCapturer::SinkWantsObserver {
+ public:
+ TemporalLayerObserver(VideoEncoderFactory* encoder_factory,
+ const std::string& payload_name,
+ const std::vector<int>& num_temporal_layers)
+ : EndToEndTest(kDefaultTimeoutMs),
+ encoder_factory_(encoder_factory),
+ payload_name_(payload_name),
+ num_temporal_layers_(num_temporal_layers),
+ depacketizer_(CreateVideoRtpDepacketizer(
+ PayloadStringToCodecType(payload_name))) {}
+
+ private:
+ void OnFrameGeneratorCapturerCreated(
+ test::FrameGeneratorCapturer* frame_generator_capturer) override {
+ frame_generator_capturer->ChangeResolution(640, 360);
+ }
+
+ void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
+ const rtc::VideoSinkWants& wants) override {}
+
+ void ModifySenderBitrateConfig(
+ BitrateConstraints* bitrate_config) override {
+ bitrate_config->start_bitrate_bps = kMaxBitrateBps / 2;
+ }
+
+ size_t GetNumVideoStreams() const override {
+ return num_temporal_layers_.size();
+ }
+
+ void ModifyVideoConfigs(
+ VideoSendStream::Config* send_config,
+ std::vector<VideoReceiveStream::Config>* receive_configs,
+ VideoEncoderConfig* encoder_config) override {
+ send_config->encoder_settings.encoder_factory = encoder_factory_;
+ send_config->rtp.payload_name = payload_name_;
+ send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType;
+ encoder_config->video_format.name = payload_name_;
+ encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
+ encoder_config->video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ payload_name_, /*max_qp=*/56, /*is_screenshare=*/false,
+ /*conference_mode=*/false);
+ encoder_config->max_bitrate_bps = kMaxBitrateBps;
+
+ for (size_t i = 0; i < num_temporal_layers_.size(); ++i) {
+ VideoStream& stream = encoder_config->simulcast_layers[i];
+ stream.num_temporal_layers = num_temporal_layers_[i];
+ configured_num_temporal_layers_[send_config->rtp.ssrcs[i]] =
+ num_temporal_layers_[i];
+ }
+ }
+
+ struct ParsedPacket {
+ uint32_t timestamp;
+ uint32_t ssrc;
+ int temporal_idx;
+ };
+
+ bool ParsePayload(const uint8_t* packet,
+ size_t length,
+ ParsedPacket& parsed) const {
+ RtpPacket rtp_packet;
+ EXPECT_TRUE(rtp_packet.Parse(packet, length));
+
+ if (rtp_packet.payload_size() == 0) {
+ return false; // Padding packet.
+ }
+ parsed.timestamp = rtp_packet.Timestamp();
+ parsed.ssrc = rtp_packet.Ssrc();
+
+ absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
+ depacketizer_->Parse(rtp_packet.PayloadBuffer());
+ EXPECT_TRUE(parsed_payload);
+
+ if (const auto* vp8_header = absl::get_if<RTPVideoHeaderVP8>(
+ &parsed_payload->video_header.video_type_header)) {
+ parsed.temporal_idx = vp8_header->temporalIdx;
+ } else {
+ RTC_NOTREACHED();
+ }
+ return true;
+ }
+
+ Action OnSendRtp(const uint8_t* packet, size_t length) override {
+ ParsedPacket parsed;
+ if (!ParsePayload(packet, length, parsed))
+ return SEND_PACKET;
+
+ uint32_t ssrc = parsed.ssrc;
+ int temporal_idx =
+ parsed.temporal_idx == kNoTemporalIdx ? 0 : parsed.temporal_idx;
+ max_observed_tl_idxs_[ssrc] =
+ std::max(temporal_idx, max_observed_tl_idxs_[ssrc]);
+
+ if (last_observed_packet_.count(ssrc) == 0 ||
+ parsed.timestamp != last_observed_packet_[ssrc].timestamp) {
+ num_observed_frames_[ssrc]++;
+ }
+ last_observed_packet_[ssrc] = parsed;
+
+ if (HighestTemporalLayerSentPerStream())
+ observation_complete_.Set();
+
+ return SEND_PACKET;
+ }
+
+ bool HighestTemporalLayerSentPerStream() const {
+ if (num_observed_frames_.size() !=
+ configured_num_temporal_layers_.size()) {
+ return false;
+ }
+ for (const auto& num_frames : num_observed_frames_) {
+ if (num_frames.second < kMinFramesToObservePerStream) {
+ return false;
+ }
+ }
+ if (max_observed_tl_idxs_.size() !=
+ configured_num_temporal_layers_.size()) {
+ return false;
+ }
+ for (const auto& max_tl_idx : max_observed_tl_idxs_) {
+ uint32_t ssrc = max_tl_idx.first;
+ int configured_num_tls =
+ configured_num_temporal_layers_.find(ssrc)->second;
+ if (max_tl_idx.second != configured_num_tls - 1)
+ return false;
+ }
+ return true;
+ }
+
+ void PerformTest() override { EXPECT_TRUE(Wait()); }
+
+ VideoEncoderFactory* const encoder_factory_;
+ const std::string payload_name_;
+ const std::vector<int> num_temporal_layers_;
+ const std::unique_ptr<VideoRtpDepacketizer> depacketizer_;
+ // Mapped by SSRC.
+ std::map<uint32_t, int> configured_num_temporal_layers_;
+ std::map<uint32_t, int> max_observed_tl_idxs_;
+ std::map<uint32_t, int> num_observed_frames_;
+ std::map<uint32_t, ParsedPacket> last_observed_packet_;
+ } test(encoder_factory, payload_name, num_temporal_layers);
+
+ RunBaseTest(&test);
+}
+
+TEST_F(VideoSendStreamTest, TestTemporalLayersVp8) {
+ InternalEncoderFactory internal_encoder_factory;
+ test::FunctionVideoEncoderFactory encoder_factory(
+ [&internal_encoder_factory]() {
+ return std::make_unique<SimulcastEncoderAdapter>(
+ &internal_encoder_factory, SdpVideoFormat("VP8"));
+ });
+
+ TestTemporalLayers(&encoder_factory, "VP8",
+ /*num_temporal_layers=*/{2});
+}
+
+TEST_F(VideoSendStreamTest, TestTemporalLayersVp8Simulcast) {
+ InternalEncoderFactory internal_encoder_factory;
+ test::FunctionVideoEncoderFactory encoder_factory(
+ [&internal_encoder_factory]() {
+ return std::make_unique<SimulcastEncoderAdapter>(
+ &internal_encoder_factory, SdpVideoFormat("VP8"));
+ });
+
+ TestTemporalLayers(&encoder_factory, "VP8",
+ /*num_temporal_layers=*/{2, 2});
+}
+
+TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithDifferentNumTls) {
+ InternalEncoderFactory internal_encoder_factory;
+ test::FunctionVideoEncoderFactory encoder_factory(
+ [&internal_encoder_factory]() {
+ return std::make_unique<SimulcastEncoderAdapter>(
+ &internal_encoder_factory, SdpVideoFormat("VP8"));
+ });
+
+ TestTemporalLayers(&encoder_factory, "VP8",
+ /*num_temporal_layers=*/{3, 1});
+}
+
+TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithoutSimAdapter) {
+ test::FunctionVideoEncoderFactory encoder_factory(
+ []() { return VP8Encoder::Create(); });
+
+ TestTemporalLayers(&encoder_factory, "VP8",
+ /*num_temporal_layers=*/{2, 2});
+}
+
} // namespace webrtc
