Let PCF.GetRtpSenderCapabilities return codecs' scalabilityModes. Also move ScalabilityModeToString to api and add RTC_EXPORT so that Chromium can use it. Bug: chromium:986069 Change-Id: I5dbbb6de9b14ca20f3ae0630552dcd44595ad5ef Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/267780 Reviewed-by: Harald Alvestrand <hta@webrtc.org> Commit-Queue: Daniel.L (Byoungchan) Lee <daniel.l@hpcnt.com> Cr-Commit-Position: refs/heads/main@{#37444}
diff --git a/api/BUILD.gn b/api/BUILD.gn index 72a5eb8..9b73010 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn
@@ -453,8 +453,10 @@ "../rtc_base:checks", "../rtc_base:stringutils", "../rtc_base/system:rtc_export", + "video_codecs:scalability_mode", ] absl_deps = [ + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ]
diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h index 45cedfd..e311577 100644 --- a/api/rtp_parameters.h +++ b/api/rtp_parameters.h
@@ -17,11 +17,13 @@ #include <string> #include <vector> +#include "absl/container/inlined_vector.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/media_types.h" #include "api/priority.h" #include "api/rtp_transceiver_direction.h" +#include "api/video_codecs/scalability_mode.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -186,6 +188,9 @@ // TODO(deadbeef): Not implemented. bool svc_multi_stream_support = false; + // https://w3c.github.io/webrtc-svc/#dom-rtcrtpcodeccapability-scalabilitymodes + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> scalability_modes; + bool operator==(const RtpCodecCapability& o) const { return name == o.name && kind == o.kind && clock_rate == o.clock_rate && preferred_payload_type == o.preferred_payload_type && @@ -194,7 +199,8 @@ parameters == o.parameters && options == o.options && max_temporal_layer_extensions == o.max_temporal_layer_extensions && max_spatial_layer_extensions == o.max_spatial_layer_extensions && - svc_multi_stream_support == o.svc_multi_stream_support; + svc_multi_stream_support == o.svc_multi_stream_support && + scalability_modes == o.scalability_modes; } bool operator!=(const RtpCodecCapability& o) const { return !(*this == o); } };
diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index 1671af1..0b6db87 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn
@@ -14,7 +14,15 @@ rtc_source_set("scalability_mode") { visibility = [ "*" ] - sources = [ "scalability_mode.h" ] + sources = [ + "scalability_mode.cc", + "scalability_mode.h", + ] + deps = [ + "../../rtc_base:checks", + "../../rtc_base/system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_library("video_codecs_api") { @@ -144,6 +152,8 @@ "../../modules/video_coding:webrtc_vp8", "../../modules/video_coding:webrtc_vp8_scalability", ] + + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } rtc_source_set("video_encoder_factory_template_libvpx_vp9_adapter") { @@ -173,6 +183,7 @@ "../../modules/video_coding/codecs/av1:libaom_av1_encoder", "../../modules/video_coding/svc:scalability_mode_util", ] + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } rtc_library("vp8_temporal_layers_factory") {
diff --git a/api/video_codecs/scalability_mode.cc b/api/video_codecs/scalability_mode.cc new file mode 100644 index 0000000..7dc8fe3 --- /dev/null +++ b/api/video_codecs/scalability_mode.cc
@@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/scalability_mode.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode) { + switch (scalability_mode) { + case ScalabilityMode::kL1T1: + return "L1T1"; + case ScalabilityMode::kL1T2: + return "L1T2"; + case ScalabilityMode::kL1T2h: + return "L1T2h"; + case ScalabilityMode::kL1T3: + return "L1T3"; + case ScalabilityMode::kL1T3h: + return "L1T3h"; + case ScalabilityMode::kL2T1: + return "L2T1"; + case ScalabilityMode::kL2T1h: + return "L2T1h"; + case ScalabilityMode::kL2T1_KEY: + return "L2T1_KEY"; + case ScalabilityMode::kL2T2: + return "L2T2"; + case ScalabilityMode::kL2T2h: + return "L2T2h"; + case ScalabilityMode::kL2T2_KEY: + return "L2T2_KEY"; + case ScalabilityMode::kL2T2_KEY_SHIFT: + return "L2T2_KEY_SHIFT"; + case ScalabilityMode::kL2T3: + return "L2T3"; + case ScalabilityMode::kL2T3h: + return "L2T3h"; + case ScalabilityMode::kL2T3_KEY: + return "L2T3_KEY"; + case ScalabilityMode::kL3T1: + return "L3T1"; + case ScalabilityMode::kL3T1h: + return "L3T1h"; + case ScalabilityMode::kL3T1_KEY: + return "L3T1_KEY"; + case ScalabilityMode::kL3T2: + return "L3T2"; + case ScalabilityMode::kL3T2h: + return "L3T2h"; + case ScalabilityMode::kL3T2_KEY: + return "L3T2_KEY"; + case ScalabilityMode::kL3T3: + return "L3T3"; + case ScalabilityMode::kL3T3h: + return "L3T3h"; + case ScalabilityMode::kL3T3_KEY: + return "L3T3_KEY"; + case ScalabilityMode::kS2T1: + return "S2T1"; + case ScalabilityMode::kS2T3: + return "S2T3"; + case ScalabilityMode::kS3T3: + return "S3T3"; + } + RTC_CHECK_NOTREACHED(); +} + +} // namespace webrtc
diff --git a/api/video_codecs/scalability_mode.h b/api/video_codecs/scalability_mode.h index a33b9e4..397e3d3 100644 --- a/api/video_codecs/scalability_mode.h +++ b/api/video_codecs/scalability_mode.h
@@ -11,6 +11,12 @@ #ifndef API_VIDEO_CODECS_SCALABILITY_MODE_H_ #define API_VIDEO_CODECS_SCALABILITY_MODE_H_ +#include <stddef.h> +#include <stdint.h> + +#include "absl/strings/string_view.h" +#include "rtc_base/system/rtc_export.h" + namespace webrtc { // Supported scalability modes. Most applications should use the @@ -18,7 +24,7 @@ // This list of currently recognized modes is intended for the api boundary // between webrtc and injected encoders. Any application usage outside of // injected encoders is strongly discouraged. -enum class ScalabilityMode { +enum class ScalabilityMode : uint8_t { kL1T1, kL1T2, kL1T2h, @@ -48,5 +54,44 @@ kS3T3, }; +inline constexpr ScalabilityMode kAllScalabilityModes[] = { + // clang-format off + ScalabilityMode::kL1T1, + ScalabilityMode::kL1T2, + ScalabilityMode::kL1T2h, + ScalabilityMode::kL1T3, + ScalabilityMode::kL1T3h, + ScalabilityMode::kL2T1, + ScalabilityMode::kL2T1h, + ScalabilityMode::kL2T1_KEY, + ScalabilityMode::kL2T2, + ScalabilityMode::kL2T2h, + ScalabilityMode::kL2T2_KEY, + ScalabilityMode::kL2T2_KEY_SHIFT, + ScalabilityMode::kL2T3, + ScalabilityMode::kL2T3h, + ScalabilityMode::kL2T3_KEY, + ScalabilityMode::kL3T1, + ScalabilityMode::kL3T1h, + ScalabilityMode::kL3T1_KEY, + ScalabilityMode::kL3T2, + ScalabilityMode::kL3T2h, + ScalabilityMode::kL3T2_KEY, + ScalabilityMode::kL3T3, + ScalabilityMode::kL3T3h, + ScalabilityMode::kL3T3_KEY, + ScalabilityMode::kS2T1, + ScalabilityMode::kS2T3, + ScalabilityMode::kS3T3, + // clang-format on +}; + +inline constexpr size_t kScalabilityModeCount = + sizeof(kAllScalabilityModes) / sizeof(ScalabilityMode); + +RTC_EXPORT +absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode); + } // namespace webrtc + #endif // API_VIDEO_CODECS_SCALABILITY_MODE_H_
diff --git a/api/video_codecs/sdp_video_format.cc b/api/video_codecs/sdp_video_format.cc index 67c8913..3543806 100644 --- a/api/video_codecs/sdp_video_format.cc +++ b/api/video_codecs/sdp_video_format.cc
@@ -70,6 +70,15 @@ const Parameters& parameters) : name(name), parameters(parameters) {} +SdpVideoFormat::SdpVideoFormat( + const std::string& name, + const Parameters& parameters, + const absl::InlinedVector<ScalabilityMode, kScalabilityModeCount>& + scalability_modes) + : name(name), + parameters(parameters), + scalability_modes(scalability_modes) {} + SdpVideoFormat::SdpVideoFormat(const SdpVideoFormat&) = default; SdpVideoFormat::SdpVideoFormat(SdpVideoFormat&&) = default; SdpVideoFormat& SdpVideoFormat::operator=(const SdpVideoFormat&) = default; @@ -80,9 +89,24 @@ std::string SdpVideoFormat::ToString() const { rtc::StringBuilder builder; builder << "Codec name: " << name << ", parameters: {"; - for (const auto& kv : parameters) + for (const auto& kv : parameters) { builder << " " << kv.first << "=" << kv.second; + } + builder << " }"; + if (!scalability_modes.empty()) { + builder << ", scalability_modes: ["; + bool first = true; + for (const auto scalability_mode : scalability_modes) { + if (first) { + first = false; + } else { + builder << ", "; + } + builder << ScalabilityModeToString(scalability_mode); + } + builder << "]"; + } return builder.str(); } @@ -105,7 +129,8 @@ } bool operator==(const SdpVideoFormat& a, const SdpVideoFormat& b) { - return a.name == b.name && a.parameters == b.parameters; + return a.name == b.name && a.parameters == b.parameters && + a.scalability_modes == b.scalability_modes; } } // namespace webrtc
diff --git a/api/video_codecs/sdp_video_format.h b/api/video_codecs/sdp_video_format.h index a1e23f4..850632e 100644 --- a/api/video_codecs/sdp_video_format.h +++ b/api/video_codecs/sdp_video_format.h
@@ -14,7 +14,9 @@ #include <map> #include <string> +#include "absl/container/inlined_vector.h" #include "api/array_view.h" +#include "api/video_codecs/scalability_mode.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -26,6 +28,11 @@ explicit SdpVideoFormat(const std::string& name); SdpVideoFormat(const std::string& name, const Parameters& parameters); + SdpVideoFormat( + const std::string& name, + const Parameters& parameters, + const absl::InlinedVector<ScalabilityMode, kScalabilityModeCount>& + scalability_modes); SdpVideoFormat(const SdpVideoFormat&); SdpVideoFormat(SdpVideoFormat&&); SdpVideoFormat& operator=(const SdpVideoFormat&); @@ -51,6 +58,7 @@ std::string name; Parameters parameters; + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> scalability_modes; }; } // namespace webrtc
diff --git a/api/video_codecs/test/video_encoder_factory_template_tests.cc b/api/video_codecs/test/video_encoder_factory_template_tests.cc index b9dfac9..4c3d0cd 100644 --- a/api/video_codecs/test/video_encoder_factory_template_tests.cc +++ b/api/video_codecs/test/video_encoder_factory_template_tests.cc
@@ -17,6 +17,7 @@ #include "test/gmock.h" #include "test/gtest.h" +using ::testing::Contains; using ::testing::Each; using ::testing::Eq; using ::testing::Field; @@ -125,9 +126,12 @@ TEST(VideoEncoderFactoryTemplate, LibvpxVp8) { VideoEncoderFactoryTemplate<LibvpxVp8EncoderTemplateAdapter> factory; - const SdpVideoFormat kVp8Sdp("VP8"); - EXPECT_THAT(factory.GetSupportedFormats(), UnorderedElementsAre(kVp8Sdp)); - EXPECT_THAT(factory.CreateVideoEncoder(kVp8Sdp), Ne(nullptr)); + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats.size(), 1); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::name, "VP8")); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL1T3))); + EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); } TEST(VideoEncoderFactoryTemplate, LibvpxVp9) { @@ -135,6 +139,8 @@ auto formats = factory.GetSupportedFormats(); EXPECT_THAT(formats, Not(IsEmpty())); EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::name, "VP9"))); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL3T3_KEY)))); EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); } @@ -146,15 +152,20 @@ auto formats = factory.GetSupportedFormats(); EXPECT_THAT(formats, Not(IsEmpty())); EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::name, "H264"))); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL1T3)))); EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); } #endif // defined(WEBRTC_USE_H264) TEST(VideoEncoderFactoryTemplate, LibaomAv1) { VideoEncoderFactoryTemplate<LibaomAv1EncoderTemplateAdapter> factory; - const SdpVideoFormat kAv1Sdp("AV1"); - EXPECT_THAT(factory.GetSupportedFormats(), UnorderedElementsAre(kAv1Sdp)); - EXPECT_THAT(factory.CreateVideoEncoder(kAv1Sdp), Ne(nullptr)); + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats.size(), 1); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::name, "AV1")); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL3T3_KEY))); + EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); } } // namespace
diff --git a/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h b/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h index d71cc00..ce273ffb 100644 --- a/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h +++ b/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h
@@ -14,13 +14,17 @@ #include <memory> #include <vector> +#include "absl/container/inlined_vector.h" #include "modules/video_coding/codecs/av1/av1_svc_config.h" #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" namespace webrtc { struct LibaomAv1EncoderTemplateAdapter { static std::vector<SdpVideoFormat> SupportedFormats() { - return {SdpVideoFormat("AV1")}; + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> + scalability_modes = LibaomAv1EncoderSupportedScalabilityModes(); + return { + SdpVideoFormat("AV1", SdpVideoFormat::Parameters(), scalability_modes)}; } static std::unique_ptr<VideoEncoder> CreateEncoder(
diff --git a/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h b/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h index c00c6d1..d7d0cd4 100644 --- a/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h +++ b/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h
@@ -14,13 +14,21 @@ #include <memory> #include <vector> +#include "absl/container/inlined_vector.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" #include "modules/video_coding/codecs/vp8/vp8_scalability.h" namespace webrtc { struct LibvpxVp8EncoderTemplateAdapter { static std::vector<SdpVideoFormat> SupportedFormats() { - return {SdpVideoFormat("VP8")}; + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> + scalability_modes; + for (const auto scalability_mode : kVP8SupportedScalabilityModes) { + scalability_modes.push_back(scalability_mode); + } + + return { + SdpVideoFormat("VP8", SdpVideoFormat::Parameters(), scalability_modes)}; } static std::unique_ptr<VideoEncoder> CreateEncoder(
diff --git a/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h b/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h index 9947ec9..c10fda4 100644 --- a/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h +++ b/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h
@@ -19,7 +19,7 @@ namespace webrtc { struct LibvpxVp9EncoderTemplateAdapter { static std::vector<SdpVideoFormat> SupportedFormats() { - return SupportedVP9Codecs(); + return SupportedVP9Codecs(/*add_scalability_modes=*/true); } static std::unique_ptr<VideoEncoder> CreateEncoder(
diff --git a/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h b/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h index 9959617..5a9742a 100644 --- a/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h +++ b/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h
@@ -22,7 +22,7 @@ #if defined(WEBRTC_USE_H264) struct OpenH264EncoderTemplateAdapter { static std::vector<SdpVideoFormat> SupportedFormats() { - return SupportedH264Codecs(); + return SupportedH264Codecs(/*add_scalability_modes=*/true); } static std::unique_ptr<VideoEncoder> CreateEncoder(
diff --git a/media/BUILD.gn b/media/BUILD.gn index 0481e3f0..e533c10 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn
@@ -102,6 +102,7 @@ ] absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ]
diff --git a/media/base/codec.cc b/media/base/codec.cc index 4935e94..8204ef3 100644 --- a/media/base/codec.cc +++ b/media/base/codec.cc
@@ -304,6 +304,7 @@ VideoCodec::VideoCodec(const webrtc::SdpVideoFormat& c) : Codec(0 /* id */, c.name, kVideoCodecClockrate) { params = c.parameters; + scalability_modes = c.scalability_modes; } VideoCodec::VideoCodec(const VideoCodec& c) = default;
diff --git a/media/base/codec.h b/media/base/codec.h index 7319022..f0ed251 100644 --- a/media/base/codec.h +++ b/media/base/codec.h
@@ -16,6 +16,7 @@ #include <string> #include <vector> +#include "absl/container/inlined_vector.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/field_trials_view.h" @@ -152,6 +153,8 @@ struct RTC_EXPORT VideoCodec : public Codec { absl::optional<std::string> packetization; + absl::InlinedVector<webrtc::ScalabilityMode, webrtc::kScalabilityModeCount> + scalability_modes; // Creates a codec with the given parameters. VideoCodec(int id, const std::string& name);
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 16b8842..6b74d2b 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn
@@ -500,6 +500,7 @@ "//third_party/libyuv", ] absl_deps = [ + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -737,6 +738,7 @@ ] absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings:strings", ]
diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn index 08a86e0..91f2cdec 100644 --- a/modules/video_coding/codecs/av1/BUILD.gn +++ b/modules/video_coding/codecs/av1/BUILD.gn
@@ -23,6 +23,8 @@ "../../svc:scalability_structures", "../../svc:scalable_video_controller", ] + + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } rtc_library("dav1d_decoder") {
diff --git a/modules/video_coding/codecs/av1/av1_svc_config.cc b/modules/video_coding/codecs/av1/av1_svc_config.cc index dd66de0..43dcf96 100644 --- a/modules/video_coding/codecs/av1/av1_svc_config.cc +++ b/modules/video_coding/codecs/av1/av1_svc_config.cc
@@ -36,6 +36,17 @@ } } // namespace +absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> +LibaomAv1EncoderSupportedScalabilityModes() { + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> scalability_modes; + for (ScalabilityMode scalability_mode : kAllScalabilityModes) { + if (ScalabilityStructureConfig(scalability_mode) != absl::nullopt) { + scalability_modes.push_back(scalability_mode); + } + } + return scalability_modes; +} + bool LibaomAv1EncoderSupportsScalabilityMode(ScalabilityMode scalability_mode) { // For libaom AV1, the scalability mode is supported if we can create the // scalability structure.
diff --git a/modules/video_coding/codecs/av1/av1_svc_config.h b/modules/video_coding/codecs/av1/av1_svc_config.h index 56cc2b3..05b886b 100644 --- a/modules/video_coding/codecs/av1/av1_svc_config.h +++ b/modules/video_coding/codecs/av1/av1_svc_config.h
@@ -10,10 +10,16 @@ #ifndef MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_ #define MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_ +#include <vector> + +#include "absl/container/inlined_vector.h" #include "api/video_codecs/video_codec.h" namespace webrtc { +absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> +LibaomAv1EncoderSupportedScalabilityModes(); + bool LibaomAv1EncoderSupportsScalabilityMode(ScalabilityMode scalability_mode); // Fills `video_codec.spatialLayers` using other members.
diff --git a/modules/video_coding/codecs/h264/h264.cc b/modules/video_coding/codecs/h264/h264.cc index b77cf0d..23580d7 100644 --- a/modules/video_coding/codecs/h264/h264.cc +++ b/modules/video_coding/codecs/h264/h264.cc
@@ -14,6 +14,7 @@ #include <memory> #include <string> +#include "absl/container/inlined_vector.h" #include "absl/types/optional.h" #include "api/video_codecs/sdp_video_format.h" #include "media/base/media_constants.h" @@ -51,15 +52,23 @@ SdpVideoFormat CreateH264Format(H264Profile profile, H264Level level, - const std::string& packetization_mode) { + const std::string& packetization_mode, + bool add_scalability_modes) { const absl::optional<std::string> profile_string = H264ProfileLevelIdToString(H264ProfileLevelId(profile, level)); RTC_CHECK(profile_string); + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> scalability_modes; + if (add_scalability_modes) { + for (const auto scalability_mode : kSupportedScalabilityModes) { + scalability_modes.push_back(scalability_mode); + } + } return SdpVideoFormat( cricket::kH264CodecName, {{cricket::kH264FmtpProfileLevelId, *profile_string}, {cricket::kH264FmtpLevelAsymmetryAllowed, "1"}, - {cricket::kH264FmtpPacketizationMode, packetization_mode}}); + {cricket::kH264FmtpPacketizationMode, packetization_mode}}, + scalability_modes); } void DisableRtcUseH264() { @@ -68,7 +77,7 @@ #endif } -std::vector<SdpVideoFormat> SupportedH264Codecs() { +std::vector<SdpVideoFormat> SupportedH264Codecs(bool add_scalability_modes) { TRACE_EVENT0("webrtc", __func__); if (!IsH264CodecSupported()) return std::vector<SdpVideoFormat>(); @@ -81,17 +90,18 @@ // // We support both packetization modes 0 (mandatory) and 1 (optional, // preferred). - return { - CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, - "1"), - CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, - "0"), - CreateH264Format(H264Profile::kProfileConstrainedBaseline, - H264Level::kLevel3_1, "1"), - CreateH264Format(H264Profile::kProfileConstrainedBaseline, - H264Level::kLevel3_1, "0"), - CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "1"), - CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "0")}; + return {CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, + "1", add_scalability_modes), + CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, + "0", add_scalability_modes), + CreateH264Format(H264Profile::kProfileConstrainedBaseline, + H264Level::kLevel3_1, "1", add_scalability_modes), + CreateH264Format(H264Profile::kProfileConstrainedBaseline, + H264Level::kLevel3_1, "0", add_scalability_modes), + CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "1", + add_scalability_modes), + CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "0", + add_scalability_modes)}; } std::vector<SdpVideoFormat> SupportedH264DecoderCodecs() {
diff --git a/modules/video_coding/codecs/h264/include/h264.h b/modules/video_coding/codecs/h264/include/h264.h index ff7e168..2635b53 100644 --- a/modules/video_coding/codecs/h264/include/h264.h +++ b/modules/video_coding/codecs/h264/include/h264.h
@@ -30,7 +30,8 @@ RTC_EXPORT SdpVideoFormat CreateH264Format(H264Profile profile, H264Level level, - const std::string& packetization_mode); + const std::string& packetization_mode, + bool add_scalability_modes = false); // Set to disable the H.264 encoder/decoder implementations that are provided if // `rtc_use_h264` build flag is true (if false, this function does nothing). @@ -40,7 +41,8 @@ // Returns a vector with all supported internal H264 encode profiles that we can // negotiate in SDP, in order of preference. -std::vector<SdpVideoFormat> SupportedH264Codecs(); +std::vector<SdpVideoFormat> SupportedH264Codecs( + bool add_scalability_modes = false); // Returns a vector with all supported internal H264 decode profiles that we can // negotiate in SDP, in order of preference. This will be available for receive
diff --git a/modules/video_coding/codecs/vp8/vp8_scalability.cc b/modules/video_coding/codecs/vp8/vp8_scalability.cc index df15be0..9c7495d 100644 --- a/modules/video_coding/codecs/vp8/vp8_scalability.cc +++ b/modules/video_coding/codecs/vp8/vp8_scalability.cc
@@ -13,9 +13,7 @@ namespace webrtc { bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode) { - constexpr ScalabilityMode kSupportedScalabilityModes[] = { - ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3}; - for (const auto& entry : kSupportedScalabilityModes) { + for (const auto& entry : kVP8SupportedScalabilityModes) { if (entry == scalability_mode) { return true; }
diff --git a/modules/video_coding/codecs/vp8/vp8_scalability.h b/modules/video_coding/codecs/vp8/vp8_scalability.h index 11f018a..923f159 100644 --- a/modules/video_coding/codecs/vp8/vp8_scalability.h +++ b/modules/video_coding/codecs/vp8/vp8_scalability.h
@@ -15,6 +15,8 @@ namespace webrtc { +inline constexpr ScalabilityMode kVP8SupportedScalabilityModes[] = { + ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3}; bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode); } // namespace webrtc
diff --git a/modules/video_coding/codecs/vp9/include/vp9.h b/modules/video_coding/codecs/vp9/include/vp9.h index 9ef36b0..79d403d 100644 --- a/modules/video_coding/codecs/vp9/include/vp9.h +++ b/modules/video_coding/codecs/vp9/include/vp9.h
@@ -24,7 +24,8 @@ // Returns a vector with all supported internal VP9 profiles that we can // negotiate in SDP, in order of preference. -std::vector<SdpVideoFormat> SupportedVP9Codecs(); +std::vector<SdpVideoFormat> SupportedVP9Codecs( + bool add_scalability_modes = false); // Returns a vector with all supported internal VP9 decode profiles in order of // preference. These will be availble for receive-only connections.
diff --git a/modules/video_coding/codecs/vp9/vp9.cc b/modules/video_coding/codecs/vp9/vp9.cc index 7a7a7d6..222e57b 100644 --- a/modules/video_coding/codecs/vp9/vp9.cc +++ b/modules/video_coding/codecs/vp9/vp9.cc
@@ -12,6 +12,7 @@ #include <memory> +#include "absl/container/inlined_vector.h" #include "api/transport/field_trial_based_config.h" #include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/sdp_video_format.h" @@ -26,7 +27,7 @@ namespace webrtc { -std::vector<SdpVideoFormat> SupportedVP9Codecs() { +std::vector<SdpVideoFormat> SupportedVP9Codecs(bool add_scalability_modes) { #ifdef RTC_ENABLE_VP9 // Profile 2 might not be available on some platforms until // https://bugs.chromium.org/p/webm/issues/detail?id=1544 is solved. @@ -36,13 +37,23 @@ (vpx_codec_get_caps(vpx_codec_vp9_dx()) & VPX_CODEC_CAP_HIGHBITDEPTH) != 0; + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> scalability_modes; + if (add_scalability_modes) { + for (const auto scalability_mode : kAllScalabilityModes) { + if (ScalabilityStructureConfig(scalability_mode).has_value()) { + scalability_modes.push_back(scalability_mode); + } + } + } std::vector<SdpVideoFormat> supported_formats{SdpVideoFormat( cricket::kVp9CodecName, - {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})}; + {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}}, + scalability_modes)}; if (vpx_supports_high_bit_depth) { supported_formats.push_back(SdpVideoFormat( cricket::kVp9CodecName, - {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile2)}})); + {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile2)}}, + scalability_modes)); } return supported_formats;
diff --git a/modules/video_coding/svc/scalability_mode_util.cc b/modules/video_coding/svc/scalability_mode_util.cc index 510c9fd3..addcdd1 100644 --- a/modules/video_coding/svc/scalability_mode_util.cc +++ b/modules/video_coding/svc/scalability_mode_util.cc
@@ -83,66 +83,6 @@ return absl::nullopt; } -absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode) { - switch (scalability_mode) { - case ScalabilityMode::kL1T1: - return "L1T1"; - case ScalabilityMode::kL1T2: - return "L1T2"; - case ScalabilityMode::kL1T2h: - return "L1T2h"; - case ScalabilityMode::kL1T3: - return "L1T3"; - case ScalabilityMode::kL1T3h: - return "L1T3h"; - case ScalabilityMode::kL2T1: - return "L2T1"; - case ScalabilityMode::kL2T1h: - return "L2T1h"; - case ScalabilityMode::kL2T1_KEY: - return "L2T1_KEY"; - case ScalabilityMode::kL2T2: - return "L2T2"; - case ScalabilityMode::kL2T2h: - return "L2T2h"; - case ScalabilityMode::kL2T2_KEY: - return "L2T2_KEY"; - case ScalabilityMode::kL2T2_KEY_SHIFT: - return "L2T2_KEY_SHIFT"; - case ScalabilityMode::kL2T3: - return "L2T3"; - case ScalabilityMode::kL2T3h: - return "L2T3h"; - case ScalabilityMode::kL2T3_KEY: - return "L2T3_KEY"; - case ScalabilityMode::kL3T1: - return "L3T1"; - case ScalabilityMode::kL3T1h: - return "L3T1h"; - case ScalabilityMode::kL3T1_KEY: - return "L3T1_KEY"; - case ScalabilityMode::kL3T2: - return "L3T2"; - case ScalabilityMode::kL3T2h: - return "L3T2h"; - case ScalabilityMode::kL3T2_KEY: - return "L3T2_KEY"; - case ScalabilityMode::kL3T3: - return "L3T3"; - case ScalabilityMode::kL3T3h: - return "L3T3h"; - case ScalabilityMode::kL3T3_KEY: - return "L3T3_KEY"; - case ScalabilityMode::kS2T1: - return "S2T1"; - case ScalabilityMode::kS2T3: - return "S2T3"; - case ScalabilityMode::kS3T3: - return "S3T3"; - } - RTC_CHECK_NOTREACHED(); -} - InterLayerPredMode ScalabilityModeToInterLayerPredMode( ScalabilityMode scalability_mode) { switch (scalability_mode) {
diff --git a/modules/video_coding/svc/scalability_mode_util.h b/modules/video_coding/svc/scalability_mode_util.h index c543c0c..e9308bd 100644 --- a/modules/video_coding/svc/scalability_mode_util.h +++ b/modules/video_coding/svc/scalability_mode_util.h
@@ -21,8 +21,6 @@ absl::optional<ScalabilityMode> ScalabilityModeFromString( absl::string_view scalability_mode_string); -absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode); - InterLayerPredMode ScalabilityModeToInterLayerPredMode( ScalabilityMode scalability_mode);
diff --git a/pc/peer_connection_factory_unittest.cc b/pc/peer_connection_factory_unittest.cc index 7f71c76..b3e9e1e 100644 --- a/pc/peer_connection_factory_unittest.cc +++ b/pc/peer_connection_factory_unittest.cc
@@ -61,6 +61,7 @@ using ::testing::InvokeWithoutArgs; using ::testing::NiceMock; using ::testing::Return; +using ::testing::UnorderedElementsAre; namespace { @@ -181,10 +182,49 @@ EXPECT_GT(codec.num_channels, 0); } - void VerifyVideoCodecCapability(const webrtc::RtpCodecCapability& codec) { + void VerifyVideoCodecCapability(const webrtc::RtpCodecCapability& codec, + bool sender) { EXPECT_EQ(codec.kind, cricket::MEDIA_TYPE_VIDEO); EXPECT_FALSE(codec.name.empty()); EXPECT_GT(codec.clock_rate, 0); + if (sender) { + if (codec.name == "VP8" || codec.name == "H264") { + EXPECT_THAT(codec.scalability_modes, + UnorderedElementsAre(webrtc::ScalabilityMode::kL1T1, + webrtc::ScalabilityMode::kL1T2, + webrtc::ScalabilityMode::kL1T3)) + << "Codec: " << codec.name; + } else if (codec.name == "VP9" || codec.name == "AV1") { + EXPECT_THAT( + codec.scalability_modes, + UnorderedElementsAre( + // clang-format off + webrtc::ScalabilityMode::kL1T1, + webrtc::ScalabilityMode::kL1T2, + webrtc::ScalabilityMode::kL1T3, + webrtc::ScalabilityMode::kL2T1, + webrtc::ScalabilityMode::kL2T1h, + webrtc::ScalabilityMode::kL2T1_KEY, + webrtc::ScalabilityMode::kL2T2, + webrtc::ScalabilityMode::kL2T2_KEY, + webrtc::ScalabilityMode::kL2T2_KEY_SHIFT, + webrtc::ScalabilityMode::kL2T3, + webrtc::ScalabilityMode::kL2T3_KEY, + webrtc::ScalabilityMode::kL3T1, + webrtc::ScalabilityMode::kL3T3, + webrtc::ScalabilityMode::kL3T3_KEY, + webrtc::ScalabilityMode::kS2T1, + webrtc::ScalabilityMode::kS2T3, + webrtc::ScalabilityMode::kS3T3) + // clang-format on + ) + << "Codec: " << codec.name; + } else { + EXPECT_TRUE(codec.scalability_modes.empty()); + } + } else { + EXPECT_TRUE(codec.scalability_modes.empty()); + } } std::unique_ptr<rtc::SocketServer> socket_server_; @@ -251,7 +291,7 @@ factory_->GetRtpSenderCapabilities(cricket::MEDIA_TYPE_VIDEO); EXPECT_FALSE(video_capabilities.codecs.empty()); for (const auto& codec : video_capabilities.codecs) { - VerifyVideoCodecCapability(codec); + VerifyVideoCodecCapability(codec, true); } EXPECT_FALSE(video_capabilities.header_extensions.empty()); for (const auto& header_extension : video_capabilities.header_extensions) { @@ -284,7 +324,7 @@ factory_->GetRtpReceiverCapabilities(cricket::MEDIA_TYPE_VIDEO); EXPECT_FALSE(video_capabilities.codecs.empty()); for (const auto& codec : video_capabilities.codecs) { - VerifyVideoCodecCapability(codec); + VerifyVideoCodecCapability(codec, false); } EXPECT_FALSE(video_capabilities.header_extensions.empty()); for (const auto& header_extension : video_capabilities.header_extensions) {
diff --git a/pc/rtp_parameters_conversion.cc b/pc/rtp_parameters_conversion.cc index afba4bc..196cb79 100644 --- a/pc/rtp_parameters_conversion.cc +++ b/pc/rtp_parameters_conversion.cc
@@ -308,7 +308,18 @@ template <> void ToRtpCodecCapabilityTypeSpecific<cricket::VideoCodec>( const cricket::VideoCodec& cricket_codec, - RtpCodecCapability* codec) {} + RtpCodecCapability* codec) { + if (cricket_codec.scalability_modes.empty() || + (cricket_codec.scalability_modes.size() == 1 && + cricket_codec.scalability_modes[0] == ScalabilityMode::kL1T1)) { + // https://w3c.github.io/webrtc-svc/#dom-rtcrtpcodeccapability-scalabilitymodes + // If a codec does not support encoding of scalability modes other than + // "L1T1", then the scalabilityModes member is not provided. + return; + } + + codec->scalability_modes = cricket_codec.scalability_modes; +} template <typename C> RtpCodecCapability ToRtpCodecCapability(const C& cricket_codec) {