Add SDP negotiation support for HEVC.
This adds neccessary checks for SDP negotiation with HEVC.
Test: Manually apply the CL on Chromium and enable HEVC HW encoder,
and add HEVC profiles in rtc video decoder/encoder factory, H265 is
negotiated in SDP with correct FMTP lines added.
Bug: webrtc:13485
Change-Id: I5557b20b646cc96c5acb578521204fe10df0dcf0
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/330202
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Jianlin Qiu <jianlin.qiu@intel.com>
Cr-Commit-Position: refs/heads/main@{#41357}
diff --git a/api/video_codecs/h265_profile_tier_level.h b/api/video_codecs/h265_profile_tier_level.h
index 3056d2b..c757e54 100644
--- a/api/video_codecs/h265_profile_tier_level.h
+++ b/api/video_codecs/h265_profile_tier_level.h
@@ -101,8 +101,9 @@
// Returns true if the parameters have the same H265 profile or neither contains
// an H265 profile, otherwise false.
-bool H265IsSameProfileTierLevel(const SdpVideoFormat::Parameters& params1,
- const SdpVideoFormat::Parameters& params2);
+RTC_EXPORT bool H265IsSameProfileTierLevel(
+ const SdpVideoFormat::Parameters& params1,
+ const SdpVideoFormat::Parameters& params2);
} // namespace webrtc
diff --git a/media/base/codec.cc b/media/base/codec.cc
index c4e1c6f..05bdb7f 100644
--- a/media/base/codec.cc
+++ b/media/base/codec.cc
@@ -15,6 +15,9 @@
#include "api/audio_codecs/audio_format.h"
#include "api/video_codecs/av1_profile.h"
#include "api/video_codecs/h264_profile_level_id.h"
+#ifdef RTC_ENABLE_H265
+#include "api/video_codecs/h265_profile_tier_level.h"
+#endif
#include "api/video_codecs/vp9_profile.h"
#include "media/base/media_constants.h"
#include "rtc_base/checks.h"
@@ -41,6 +44,24 @@
GetH264PacketizationModeOrDefault(right);
}
+#ifdef RTC_ENABLE_H265
+std::string GetH265TxModeOrDefault(const CodecParameterMap& params) {
+ auto it = params.find(kH265FmtpTxMode);
+ if (it != params.end()) {
+ return it->second;
+ }
+ // If TxMode is not present, a value of "SRST" must be inferred.
+ // https://tools.ietf.org/html/rfc7798@section-7.1
+ return "SRST";
+}
+
+bool IsSameH265TxMode(const CodecParameterMap& left,
+ const CodecParameterMap& right) {
+ return absl::EqualsIgnoreCase(GetH265TxModeOrDefault(left),
+ GetH265TxModeOrDefault(right));
+}
+#endif
+
// Some (video) codecs are actually families of codecs and rely on parameters
// to distinguish different incompatible family members.
bool IsSameCodecSpecific(const std::string& name1,
@@ -59,6 +80,12 @@
return webrtc::VP9IsSameProfile(params1, params2);
if (either_name_matches(kAv1CodecName))
return webrtc::AV1IsSameProfile(params1, params2);
+#ifdef RTC_ENABLE_H265
+ if (either_name_matches(kH265CodecName)) {
+ return webrtc::H265IsSameProfileTierLevel(params1, params2) &&
+ IsSameH265TxMode(params1, params2);
+ }
+#endif
return true;
}
diff --git a/media/base/codec.h b/media/base/codec.h
index bd4239b..bc8b841 100644
--- a/media/base/codec.h
+++ b/media/base/codec.h
@@ -98,6 +98,9 @@
absl::InlinedVector<webrtc::ScalabilityMode, webrtc::kScalabilityModeCount>
scalability_modes;
+ // H.265 only
+ absl::optional<std::string> tx_mode;
+
// Non key-value parameters such as the telephone-event "0‐15" are
// represented using an empty string as key, i.e. {"": "0-15"}.
CodecParameterMap params;
@@ -110,7 +113,9 @@
// Indicates if this codec is compatible with the specified codec by
// checking the assigned id and profile values for the relevant video codecs.
- // H264 levels are not compared.
+ // For H.264, packetization modes will be compared; If H.265 is enabled,
+ // TxModes will be compared.
+ // H.264(and H.265, if enabled) levels are not compared.
bool Matches(const Codec& codec) const;
bool MatchesRtpCodec(const webrtc::RtpCodec& capability) const;
diff --git a/media/base/codec_unittest.cc b/media/base/codec_unittest.cc
index eb34530..4dc3b18 100644
--- a/media/base/codec_unittest.cc
+++ b/media/base/codec_unittest.cc
@@ -342,6 +342,67 @@
}
}
+#ifdef RTC_ENABLE_H265
+// Matching H.265 codecs should have matching profile/tier/level and tx-mode.
+TEST(CodecTest, TestH265CodecMatches) {
+ constexpr char kProfile1[] = "1";
+ constexpr char kTier1[] = "1";
+ constexpr char kLevel3_1[] = "93";
+ constexpr char kLevel4[] = "120";
+ constexpr char kTxMrst[] = "MRST";
+
+ VideoCodec c_ptl_blank =
+ cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+
+ {
+ VideoCodec c_profile_1 =
+ cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+ c_profile_1.params[cricket::kH265FmtpProfileId] = kProfile1;
+
+ // Matches since profile-id unspecified defaults to "1".
+ EXPECT_TRUE(c_ptl_blank.Matches(c_profile_1));
+ }
+
+ {
+ VideoCodec c_tier_flag_1 =
+ cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+ c_tier_flag_1.params[cricket::kH265FmtpTierFlag] = kTier1;
+
+ // Does not match since profile-space unspecified defaults to "0".
+ EXPECT_FALSE(c_ptl_blank.Matches(c_tier_flag_1));
+ }
+
+ {
+ VideoCodec c_level_id_3_1 =
+ cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+ c_level_id_3_1.params[cricket::kH265FmtpLevelId] = kLevel3_1;
+
+ // Matches since level-id unspecified defautls to "93".
+ EXPECT_TRUE(c_ptl_blank.Matches(c_level_id_3_1));
+ }
+
+ {
+ VideoCodec c_level_id_4 =
+ cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+ c_level_id_4.params[cricket::kH265FmtpLevelId] = kLevel4;
+
+ // Does not match since different level-ids are specified.
+ EXPECT_FALSE(c_ptl_blank.Matches(c_level_id_4));
+ }
+
+ {
+ VideoCodec c_tx_mode_mrst =
+ cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+ c_tx_mode_mrst.params[cricket::kH265FmtpTxMode] = kTxMrst;
+
+ // Does not match since tx-mode implies to "SRST" and must be not specified
+ // when it is the only mode supported:
+ // https://datatracker.ietf.org/doc/html/draft-ietf-avtcore-hevc-webrtc
+ EXPECT_FALSE(c_ptl_blank.Matches(c_tx_mode_mrst));
+ }
+}
+#endif
+
TEST(CodecTest, TestSetParamGetParamAndRemoveParam) {
AudioCodec codec = cricket::CreateAudioCodec(0, "foo", 22222, 2);
codec.SetParam("a", "1");
diff --git a/media/base/sdp_video_format_utils.cc b/media/base/sdp_video_format_utils.cc
index a156afd..88d6d58 100644
--- a/media/base/sdp_video_format_utils.cc
+++ b/media/base/sdp_video_format_utils.cc
@@ -15,6 +15,9 @@
#include <utility>
#include "api/video_codecs/h264_profile_level_id.h"
+#ifdef RTC_ENABLE_H265
+#include "api/video_codecs/h265_profile_tier_level.h"
+#endif
#include "rtc_base/checks.h"
#include "rtc_base/string_to_number.h"
@@ -27,6 +30,11 @@
// Max frame size for VP8 and VP9 video.
const char kVPxFmtpMaxFrameSize[] = "max-fs";
const int kVPxFmtpFrameSizeSubBlockPixels = 256;
+#ifdef RTC_ENABLE_H265
+constexpr char kH265ProfileId[] = "profile-id";
+constexpr char kH265TierFlag[] = "tier-flag";
+constexpr char kH265LevelId[] = "level-id";
+#endif
bool IsH264LevelAsymmetryAllowed(const SdpVideoFormat::Parameters& params) {
const auto it = params.find(kH264LevelAsymmetryAllowed);
@@ -60,8 +68,59 @@
return i;
}
+#ifdef RTC_ENABLE_H265
+// Compares two H265Level and return the smaller.
+H265Level H265LevelMin(H265Level a, H265Level b) {
+ return a <= b ? a : b;
+}
+
+// Returns true if none of profile-id/tier-flag/level-id is specified
+// explicitly in the param.
+bool IsDefaultH265PTL(const SdpVideoFormat::Parameters& params) {
+ return !params.count(kH265ProfileId) && !params.count(kH265TierFlag) &&
+ !params.count(kH265LevelId);
+}
+#endif
+
} // namespace
+#ifdef RTC_ENABLE_H265
+// Set level according to https://tools.ietf.org/html/rfc7798#section-7.1
+void H265GenerateProfileTierLevelForAnswer(
+ const SdpVideoFormat::Parameters& local_supported_params,
+ const SdpVideoFormat::Parameters& remote_offered_params,
+ SdpVideoFormat::Parameters* answer_params) {
+ // If local and remote haven't set profile-id/tier-flag/level-id, they
+ // are both using the default PTL In this case, don't set PTL in answer
+ // either.
+ if (IsDefaultH265PTL(local_supported_params) &&
+ IsDefaultH265PTL(remote_offered_params)) {
+ return;
+ }
+
+ // Parse profile-tier-level.
+ const absl::optional<H265ProfileTierLevel> local_profile_tier_level =
+ ParseSdpForH265ProfileTierLevel(local_supported_params);
+ const absl::optional<H265ProfileTierLevel> remote_profile_tier_level =
+ ParseSdpForH265ProfileTierLevel(remote_offered_params);
+ // Profile and tier for local and remote codec must be valid and equal.
+ RTC_DCHECK(local_profile_tier_level);
+ RTC_DCHECK(remote_profile_tier_level);
+ RTC_DCHECK_EQ(local_profile_tier_level->profile,
+ remote_profile_tier_level->profile);
+ RTC_DCHECK_EQ(local_profile_tier_level->tier,
+ remote_profile_tier_level->tier);
+
+ const H265Level answer_level = H265LevelMin(local_profile_tier_level->level,
+ remote_profile_tier_level->level);
+
+ // Level-id in answer is changable as long as the highest level indicated by
+ // the answer is not higher than that indicated by the offer. See
+ // https://tools.ietf.org/html/rfc7798#section-7.2.2, sub-clause 2.
+ (*answer_params)[kH265LevelId] = H265LevelToString(answer_level);
+}
+#endif
+
// Set level according to https://tools.ietf.org/html/rfc6184#section-8.2.2.
void H264GenerateProfileLevelIdForAnswer(
const SdpVideoFormat::Parameters& local_supported_params,
diff --git a/media/base/sdp_video_format_utils.h b/media/base/sdp_video_format_utils.h
index 80c1e4d..421cab8 100644
--- a/media/base/sdp_video_format_utils.h
+++ b/media/base/sdp_video_format_utils.h
@@ -36,6 +36,18 @@
const SdpVideoFormat::Parameters& remote_offered_params,
SdpVideoFormat::Parameters* answer_params);
+#ifdef RTC_ENABLE_H265
+// Works similarly as H264GenerateProfileLevelIdForAnswer, but generates codec
+// parameters that will be used as answer for H.265.
+// Media configuration parameters, except level-id, must be used symmetrically.
+// For level-id, the highest level indicated by the answer must not be higher
+// than that indicated by the offer.
+void H265GenerateProfileTierLevelForAnswer(
+ const SdpVideoFormat::Parameters& local_supported_params,
+ const SdpVideoFormat::Parameters& remote_offered_params,
+ SdpVideoFormat::Parameters* answer_params);
+#endif
+
// Parse max frame rate from SDP FMTP line. absl::nullopt is returned if the
// field is missing or not a number.
absl::optional<int> ParseSdpForVPxMaxFrameRate(
diff --git a/media/base/sdp_video_format_utils_unittest.cc b/media/base/sdp_video_format_utils_unittest.cc
index d8ef9ab..3ae2b3bf 100644
--- a/media/base/sdp_video_format_utils_unittest.cc
+++ b/media/base/sdp_video_format_utils_unittest.cc
@@ -72,6 +72,37 @@
EXPECT_EQ("42e01f", answer_params["profile-level-id"]);
}
+#ifdef RTC_ENABLE_H265
+// Answer should not include explicit PTL info if neither local nor remote set
+// any of them.
+TEST(SdpVideoFormatUtilsTest, H265GenerateProfileTierLevelEmpty) {
+ SdpVideoFormat::Parameters answer_params;
+ H265GenerateProfileTierLevelForAnswer(SdpVideoFormat::Parameters(),
+ SdpVideoFormat::Parameters(),
+ &answer_params);
+ EXPECT_TRUE(answer_params.empty());
+}
+
+// Answer must use the minimum level as supported by both local and remote.
+TEST(SdpVideoFormatUtilsTest, H265GenerateProfileTierLevelNoEmpty) {
+ constexpr char kLocallySupportedLevelId[] = "93";
+ constexpr char kRemoteOfferedLevelId[] = "120";
+
+ SdpVideoFormat::Parameters local_params;
+ local_params["profile-id"] = "1";
+ local_params["tier-flag"] = "0";
+ local_params["level-id"] = kLocallySupportedLevelId;
+ SdpVideoFormat::Parameters remote_params;
+ remote_params["profile-id"] = "1";
+ remote_params["tier-flag"] = "0";
+ remote_params["level-id"] = kRemoteOfferedLevelId;
+ SdpVideoFormat::Parameters answer_params;
+ H265GenerateProfileTierLevelForAnswer(local_params, remote_params,
+ &answer_params);
+ EXPECT_EQ(kLocallySupportedLevelId, answer_params["level-id"]);
+}
+#endif
+
TEST(SdpVideoFormatUtilsTest, MaxFrameRateIsMissingOrInvalid) {
SdpVideoFormat::Parameters params;
absl::optional<int> empty = ParseSdpForVPxMaxFrameRate(params);
diff --git a/media/engine/internal_decoder_factory_unittest.cc b/media/engine/internal_decoder_factory_unittest.cc
index bb2e24d..51d6a94 100644
--- a/media/engine/internal_decoder_factory_unittest.cc
+++ b/media/engine/internal_decoder_factory_unittest.cc
@@ -43,6 +43,8 @@
#else
constexpr bool kDav1dIsIncluded = false;
#endif
+constexpr bool kH265Enabled = false;
+
constexpr VideoDecoderFactory::CodecSupport kSupported = {
/*is_supported=*/true, /*is_power_efficient=*/false};
constexpr VideoDecoderFactory::CodecSupport kUnsupported = {
@@ -99,6 +101,14 @@
}
}
+// At current stage since internal H.265 decoder is not implemented,
+TEST(InternalDecoderFactoryTest, H265IsNotEnabled) {
+ InternalDecoderFactory factory;
+ std::unique_ptr<VideoDecoder> decoder =
+ factory.CreateVideoDecoder(SdpVideoFormat(cricket::kH265CodecName));
+ EXPECT_EQ(static_cast<bool>(decoder), kH265Enabled);
+}
+
#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY)
TEST(InternalDecoderFactoryTest, Av1) {
InternalDecoderFactory factory;
diff --git a/media/engine/internal_encoder_factory_unittest.cc b/media/engine/internal_encoder_factory_unittest.cc
index a1c90b8..b9ca6d8 100644
--- a/media/engine/internal_encoder_factory_unittest.cc
+++ b/media/engine/internal_encoder_factory_unittest.cc
@@ -33,6 +33,8 @@
#else
constexpr bool kH264Enabled = false;
#endif
+constexpr bool kH265Enabled = false;
+
constexpr VideoEncoderFactory::CodecSupport kSupported = {
/*is_supported=*/true, /*is_power_efficient=*/false};
constexpr VideoEncoderFactory::CodecSupport kUnsupported = {
@@ -78,6 +80,17 @@
}
}
+// At current stage H.265 is not supported by internal encoder factory.
+TEST(InternalEncoderFactoryTest, H265IsNotEnabled) {
+ InternalEncoderFactory factory;
+ std::unique_ptr<VideoEncoder> encoder =
+ factory.CreateVideoEncoder(SdpVideoFormat(cricket::kH265CodecName));
+ EXPECT_EQ(static_cast<bool>(encoder), kH265Enabled);
+ EXPECT_THAT(
+ factory.GetSupportedFormats(),
+ Not(Contains(Field(&SdpVideoFormat::name, cricket::kH265CodecName))));
+}
+
TEST(InternalEncoderFactoryTest, QueryCodecSupportWithScalabilityMode) {
InternalEncoderFactory factory;
// VP8 and VP9 supported for singles spatial layers.
diff --git a/pc/media_session.cc b/pc/media_session.cc
index 573e352..e3197c4 100644
--- a/pc/media_session.cc
+++ b/pc/media_session.cc
@@ -728,6 +728,16 @@
: absl::nullopt;
}
+#ifdef RTC_ENABLE_H265
+void NegotiateTxMode(const Codec& local_codec,
+ const Codec& remote_codec,
+ Codec* negotiated_codec) {
+ negotiated_codec->tx_mode = (local_codec.tx_mode == remote_codec.tx_mode)
+ ? local_codec.tx_mode
+ : absl::nullopt;
+}
+#endif
+
// Finds a codec in `codecs2` that matches `codec_to_match`, which is
// a member of `codecs1`. If `codec_to_match` is an RED or RTX codec, both
// the codecs themselves and their associated codecs must match.
@@ -849,6 +859,13 @@
webrtc::H264GenerateProfileLevelIdForAnswer(ours.params, theirs->params,
&negotiated.params);
}
+#ifdef RTC_ENABLE_H265
+ if (absl::EqualsIgnoreCase(ours.name, kH265CodecName)) {
+ webrtc::H265GenerateProfileTierLevelForAnswer(
+ ours.params, theirs->params, &negotiated.params);
+ NegotiateTxMode(ours, *theirs, &negotiated);
+ }
+#endif
negotiated.id = theirs->id;
negotiated.name = theirs->name;
negotiated_codecs->push_back(std::move(negotiated));
diff --git a/pc/media_session_unittest.cc b/pc/media_session_unittest.cc
index 641f638..f4fd09c 100644
--- a/pc/media_session_unittest.cc
+++ b/pc/media_session_unittest.cc
@@ -4323,6 +4323,80 @@
EXPECT_EQ(vcd1->codecs()[0].id, vcd2->codecs()[0].id);
}
+#ifdef RTC_ENABLE_H265
+// Test verifying that negotiating codecs with the same tx-mode retains the
+// tx-mode value.
+TEST_F(MediaSessionDescriptionFactoryTest, H265TxModeIsEqualRetainIt) {
+ std::vector f1_codecs = {CreateVideoCodec(96, "H265")};
+ f1_codecs.back().tx_mode = "mrst";
+ f1_.set_video_codecs(f1_codecs, f1_codecs);
+
+ std::vector f2_codecs = {CreateVideoCodec(96, "H265")};
+ f2_codecs.back().tx_mode = "mrst";
+ f2_.set_video_codecs(f2_codecs, f2_codecs);
+
+ MediaSessionOptions opts;
+ AddMediaDescriptionOptions(MEDIA_TYPE_VIDEO, "video1",
+ RtpTransceiverDirection::kSendRecv, kActive,
+ &opts);
+
+ // Create an offer with two video sections using same codecs.
+ std::unique_ptr<SessionDescription> offer =
+ f1_.CreateOfferOrError(opts, nullptr).MoveValue();
+ ASSERT_TRUE(offer);
+ ASSERT_EQ(1u, offer->contents().size());
+ const MediaContentDescription* vcd1 =
+ offer->contents()[0].media_description();
+ ASSERT_EQ(1u, vcd1->codecs().size());
+ EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst");
+
+ // Create answer and negotiate the codecs.
+ std::unique_ptr<SessionDescription> answer =
+ f2_.CreateAnswerOrError(offer.get(), opts, nullptr).MoveValue();
+ ASSERT_TRUE(answer);
+ ASSERT_EQ(1u, answer->contents().size());
+ vcd1 = answer->contents()[0].media_description();
+ ASSERT_EQ(1u, vcd1->codecs().size());
+ EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst");
+}
+
+// Test verifying that negotiating codecs with different tx_mode removes
+// the tx_mode value.
+TEST_F(MediaSessionDescriptionFactoryTest, H265TxModeIsDifferentDropCodecs) {
+ std::vector f1_codecs = {CreateVideoCodec(96, "H265")};
+ f1_codecs.back().tx_mode = "mrst";
+ f1_.set_video_codecs(f1_codecs, f1_codecs);
+
+ std::vector f2_codecs = {CreateVideoCodec(96, "H265")};
+ f2_codecs.back().tx_mode = "mrmt";
+ f2_.set_video_codecs(f2_codecs, f2_codecs);
+
+ MediaSessionOptions opts;
+ AddMediaDescriptionOptions(MEDIA_TYPE_VIDEO, "video1",
+ RtpTransceiverDirection::kSendRecv, kActive,
+ &opts);
+
+ // Create an offer with two video sections using same codecs.
+ std::unique_ptr<SessionDescription> offer =
+ f1_.CreateOfferOrError(opts, nullptr).MoveValue();
+ ASSERT_TRUE(offer);
+ ASSERT_EQ(1u, offer->contents().size());
+ const VideoContentDescription* vcd1 =
+ offer->contents()[0].media_description()->as_video();
+ ASSERT_EQ(1u, vcd1->codecs().size());
+ EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst");
+
+ // Create answer and negotiate the codecs.
+ std::unique_ptr<SessionDescription> answer =
+ f2_.CreateAnswerOrError(offer.get(), opts, nullptr).MoveValue();
+ ASSERT_TRUE(answer);
+ ASSERT_EQ(1u, answer->contents().size());
+ vcd1 = answer->contents()[0].media_description()->as_video();
+ ASSERT_EQ(1u, vcd1->codecs().size());
+ EXPECT_EQ(vcd1->codecs()[0].tx_mode, absl::nullopt);
+}
+#endif
+
// Test verifying that negotiating codecs with the same packetization retains
// the packetization value.
TEST_F(MediaSessionDescriptionFactoryTest, PacketizationIsEqual) {