Add RTP header extension for HDR metadata
Bug: webrtc:8651
Change-Id: I1c956eaac1532ac0d3820084edb4054a4cc9c68d
Reviewed-on: https://webrtc-review.googlesource.com/c/109924
Commit-Queue: Johannes Kron <kron@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Alex Loiko <aleloi@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25578}diff --git a/api/rtp_headers.h b/api/rtp_headers.h
index eff6223..3e51f43 100644
--- a/api/rtp_headers.h
+++ b/api/rtp_headers.h
@@ -15,7 +15,9 @@
#include <stdint.h>
#include <string.h>
+#include "absl/types/optional.h"
#include "api/array_view.h"
+#include "api/video/hdr_metadata.h"
#include "api/video/video_content_type.h"
#include "api/video/video_frame_marking.h"
#include "api/video/video_rotation.h"
@@ -126,6 +128,8 @@
// For identifying the media section used to interpret this RTP packet. See
// https://tools.ietf.org/html/draft-ietf-mmusic-sdp-bundle-negotiation-38
Mid mid;
+
+ absl::optional<HdrMetadata> hdr_metadata;
};
struct RTPHeader {
diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h
index 5b9a051..138ed0a 100644
--- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h
+++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h
@@ -112,6 +112,7 @@
kRtpExtensionRepairedRtpStreamId,
kRtpExtensionMid,
kRtpExtensionGenericFrameDescriptor,
+ kRtpExtensionHdrMetadata,
kRtpExtensionNumberOfExtensions // Must be the last entity in the enum.
};
diff --git a/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/modules/rtp_rtcp/source/rtp_header_extension_map.cc
index dde25e3..49857a0 100644
--- a/modules/rtp_rtcp/source/rtp_header_extension_map.cc
+++ b/modules/rtp_rtcp/source/rtp_header_extension_map.cc
@@ -43,6 +43,7 @@
CreateExtensionInfo<RepairedRtpStreamId>(),
CreateExtensionInfo<RtpMid>(),
CreateExtensionInfo<RtpGenericFrameDescriptorExtension>(),
+ CreateExtensionInfo<HdrMetadataExtension>(),
};
// Because of kRtpExtensionNone, NumberOfExtension is 1 bigger than the actual
diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.cc b/modules/rtp_rtcp/source/rtp_header_extensions.cc
index 082e0e0..fe327b3 100644
--- a/modules/rtp_rtcp/source/rtp_header_extensions.cc
+++ b/modules/rtp_rtcp/source/rtp_header_extensions.cc
@@ -11,6 +11,7 @@
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include <string.h>
+#include <cmath>
#include "modules/rtp_rtcp/include/rtp_cvo.h"
#include "modules/rtp_rtcp/source/byte_io.h"
@@ -433,6 +434,135 @@
return true;
}
+// HDR Metadata.
+//
+// RTP header extension to carry HDR metadata.
+// Float values are upscaled by a static factor and transmitted as integers.
+//
+// 0 1 2 3
+// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | ID | length | luminance_max |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | | luminance_min |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | mastering_metadata.primary_r.x and .y |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | mastering_metadata.primary_g.x and .y |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | mastering_metadata.primary_b.x and .y |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | mastering_metadata.white.x and .y |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | max_content_light_level |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | max_frame_average_light_level |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+constexpr RTPExtensionType HdrMetadataExtension::kId;
+constexpr uint8_t HdrMetadataExtension::kValueSizeBytes;
+constexpr const char HdrMetadataExtension::kUri[];
+
+bool HdrMetadataExtension::Parse(rtc::ArrayView<const uint8_t> data,
+ HdrMetadata* hdr_metadata) {
+ RTC_DCHECK(hdr_metadata);
+ if (data.size() != kValueSizeBytes)
+ return false;
+
+ size_t offset = 0;
+ offset += ParseLuminance(data.data() + offset,
+ &hdr_metadata->mastering_metadata.luminance_max,
+ kLuminanceMaxDenominator);
+ offset += ParseLuminance(data.data() + offset,
+ &hdr_metadata->mastering_metadata.luminance_min,
+ kLuminanceMinDenominator);
+ offset += ParseChromaticity(data.data() + offset,
+ &hdr_metadata->mastering_metadata.primary_r);
+ offset += ParseChromaticity(data.data() + offset,
+ &hdr_metadata->mastering_metadata.primary_g);
+ offset += ParseChromaticity(data.data() + offset,
+ &hdr_metadata->mastering_metadata.primary_b);
+ offset += ParseChromaticity(data.data() + offset,
+ &hdr_metadata->mastering_metadata.white_point);
+ // TODO(kron): Do we need 32 bit here or is it enough with 16 bits?
+ // Also, what resolution is needed?
+ hdr_metadata->max_content_light_level =
+ ByteReader<uint32_t>::ReadBigEndian(data.data() + offset);
+ offset += 4;
+ hdr_metadata->max_frame_average_light_level =
+ ByteReader<uint32_t>::ReadBigEndian(data.data() + offset);
+ RTC_DCHECK_EQ(kValueSizeBytes, offset + 4);
+ return true;
+}
+
+bool HdrMetadataExtension::Write(rtc::ArrayView<uint8_t> data,
+ const HdrMetadata& hdr_metadata) {
+ RTC_DCHECK_EQ(data.size(), kValueSizeBytes);
+ size_t offset = 0;
+ offset += WriteLuminance(data.data() + offset,
+ hdr_metadata.mastering_metadata.luminance_max,
+ kLuminanceMaxDenominator);
+ offset += WriteLuminance(data.data() + offset,
+ hdr_metadata.mastering_metadata.luminance_min,
+ kLuminanceMinDenominator);
+ offset += WriteChromaticity(data.data() + offset,
+ hdr_metadata.mastering_metadata.primary_r);
+ offset += WriteChromaticity(data.data() + offset,
+ hdr_metadata.mastering_metadata.primary_g);
+ offset += WriteChromaticity(data.data() + offset,
+ hdr_metadata.mastering_metadata.primary_b);
+ offset += WriteChromaticity(data.data() + offset,
+ hdr_metadata.mastering_metadata.white_point);
+
+ // TODO(kron): Do we need 32 bit here or is it enough with 16 bits?
+ // Also, what resolution is needed?
+ ByteWriter<uint32_t>::WriteBigEndian(data.data() + offset,
+ hdr_metadata.max_content_light_level);
+ offset += 4;
+ ByteWriter<uint32_t>::WriteBigEndian(
+ data.data() + offset, hdr_metadata.max_frame_average_light_level);
+ RTC_DCHECK_EQ(kValueSizeBytes, offset + 4);
+ return true;
+}
+
+size_t HdrMetadataExtension::ParseChromaticity(
+ const uint8_t* data,
+ HdrMasteringMetadata::Chromaticity* p) {
+ uint16_t chromaticity_x_scaled = ByteReader<uint16_t>::ReadBigEndian(data);
+ uint16_t chromaticity_y_scaled =
+ ByteReader<uint16_t>::ReadBigEndian(data + 2);
+ p->x = static_cast<float>(chromaticity_x_scaled) / kChromaticityDenominator;
+ p->y = static_cast<float>(chromaticity_y_scaled) / kChromaticityDenominator;
+ return 4; // Return number of bytes read.
+}
+
+size_t HdrMetadataExtension::ParseLuminance(const uint8_t* data,
+ float* f,
+ int denominator) {
+ uint32_t luminance_scaled = ByteReader<uint32_t, 3>::ReadBigEndian(data);
+ *f = static_cast<float>(luminance_scaled) / denominator;
+ return 3; // Return number of bytes read.
+}
+
+size_t HdrMetadataExtension::WriteChromaticity(
+ uint8_t* data,
+ const HdrMasteringMetadata::Chromaticity& p) {
+ RTC_DCHECK_GE(p.x, 0.0f);
+ RTC_DCHECK_GE(p.y, 0.0f);
+ ByteWriter<uint16_t>::WriteBigEndian(
+ data, std::round(p.x * kChromaticityDenominator));
+ ByteWriter<uint16_t>::WriteBigEndian(
+ data + 2, std::round(p.y * kChromaticityDenominator));
+ return 4; // Return number of bytes written.
+}
+
+size_t HdrMetadataExtension::WriteLuminance(uint8_t* data,
+ float f,
+ int denominator) {
+ RTC_DCHECK_GE(f, 0.0f);
+ ByteWriter<uint32_t, 3>::WriteBigEndian(data, std::round(f * denominator));
+ return 3; // Return number of bytes written.
+}
+
bool BaseRtpStringExtension::Parse(rtc::ArrayView<const uint8_t> data,
StringRtpHeaderExtension* str) {
if (data.empty() || data[0] == 0) // Valid string extension can't be empty.
diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h
index 808356a..ba43415 100644
--- a/modules/rtp_rtcp/source/rtp_header_extensions.h
+++ b/modules/rtp_rtcp/source/rtp_header_extensions.h
@@ -16,6 +16,7 @@
#include "api/array_view.h"
#include "api/rtp_headers.h"
+#include "api/video/hdr_metadata.h"
#include "api/video/video_content_type.h"
#include "api/video/video_frame_marking.h"
#include "api/video/video_rotation.h"
@@ -181,6 +182,32 @@
static bool IsScalable(uint8_t temporal_id, uint8_t layer_id);
};
+class HdrMetadataExtension {
+ public:
+ using value_type = HdrMetadata;
+ static constexpr RTPExtensionType kId = kRtpExtensionHdrMetadata;
+ static constexpr uint8_t kValueSizeBytes = 30;
+ // TODO(webrtc:8651): Change to a valid uri.
+ static constexpr const char kUri[] = "rtp-hdr-metadata-uri-placeholder";
+
+ static bool Parse(rtc::ArrayView<const uint8_t> data,
+ HdrMetadata* hdr_metadata);
+ static size_t ValueSize(const HdrMetadata&) { return kValueSizeBytes; }
+ static bool Write(rtc::ArrayView<uint8_t> data,
+ const HdrMetadata& hdr_metadata);
+
+ private:
+ static constexpr int kChromaticityDenominator = 10000; // 0.0001 resolution.
+ static constexpr int kLuminanceMaxDenominator = 100; // 0.01 resolution.
+ static constexpr int kLuminanceMinDenominator = 10000; // 0.0001 resolution.
+ static size_t ParseChromaticity(const uint8_t* data,
+ HdrMasteringMetadata::Chromaticity* p);
+ static size_t ParseLuminance(const uint8_t* data, float* f, int denominator);
+ static size_t WriteChromaticity(uint8_t* data,
+ const HdrMasteringMetadata::Chromaticity& p);
+ static size_t WriteLuminance(uint8_t* data, float f, int denominator);
+};
+
// Base extension class for RTP header extensions which are strings.
// Subclasses must defined kId and kUri static constexpr members.
class BaseRtpStringExtension {
diff --git a/modules/rtp_rtcp/source/rtp_packet_received.cc b/modules/rtp_rtcp/source/rtp_packet_received.cc
index 93c0a1e..ff7b4e8 100644
--- a/modules/rtp_rtcp/source/rtp_packet_received.cc
+++ b/modules/rtp_rtcp/source/rtp_packet_received.cc
@@ -69,6 +69,7 @@
GetExtension<RepairedRtpStreamId>(&header->extension.repaired_stream_id);
GetExtension<RtpMid>(&header->extension.mid);
GetExtension<PlayoutDelayLimits>(&header->extension.playout_delay);
+ header->extension.hdr_metadata = GetExtension<HdrMetadataExtension>();
}
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_packet_unittest.cc b/modules/rtp_rtcp/source/rtp_packet_unittest.cc
index b485df6..37a9a53 100644
--- a/modules/rtp_rtcp/source/rtp_packet_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_packet_unittest.cc
@@ -185,6 +185,24 @@
0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00};
// clang-format on
+
+HdrMetadata CreateTestHdrMetadata() {
+ // Random but reasonable HDR metadata.
+ HdrMetadata hdr_metadata;
+ hdr_metadata.mastering_metadata.luminance_max = 2000.0;
+ hdr_metadata.mastering_metadata.luminance_min = 2.0001;
+ hdr_metadata.mastering_metadata.primary_r.x = 0.3003;
+ hdr_metadata.mastering_metadata.primary_r.y = 0.4004;
+ hdr_metadata.mastering_metadata.primary_g.x = 0.3201;
+ hdr_metadata.mastering_metadata.primary_g.y = 0.4604;
+ hdr_metadata.mastering_metadata.primary_b.x = 0.3409;
+ hdr_metadata.mastering_metadata.primary_b.y = 0.4907;
+ hdr_metadata.mastering_metadata.white_point.x = 0.4103;
+ hdr_metadata.mastering_metadata.white_point.y = 0.4806;
+ hdr_metadata.max_content_light_level = 2345;
+ hdr_metadata.max_frame_average_light_level = 1789;
+ return hdr_metadata;
+}
} // namespace
TEST(RtpPacketTest, CreateMinimum) {
@@ -801,4 +819,21 @@
EXPECT_EQ(receivied_timing.flags, 0);
}
+TEST(RtpPacketTest, CreateAndParseHdrMetadataExtension) {
+ // Create packet with extension.
+ RtpPacket::ExtensionManager extensions(/*extmap-allow-mixed=*/true);
+ extensions.Register<HdrMetadataExtension>(1);
+ RtpPacket packet(&extensions);
+ const HdrMetadata kHdrMetadata = CreateTestHdrMetadata();
+ EXPECT_TRUE(packet.SetExtension<HdrMetadataExtension>(kHdrMetadata));
+ packet.SetPayloadSize(42);
+
+ // Read packet with the extension.
+ RtpPacketReceived parsed(&extensions);
+ EXPECT_TRUE(parsed.Parse(packet.Buffer()));
+ HdrMetadata parsed_hdr_metadata;
+ EXPECT_TRUE(parsed.GetExtension<HdrMetadataExtension>(&parsed_hdr_metadata));
+ EXPECT_EQ(kHdrMetadata, parsed_hdr_metadata);
+}
+
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_utility.cc b/modules/rtp_rtcp/source/rtp_utility.cc
index 53a006d..80f0eab 100644
--- a/modules/rtp_rtcp/source/rtp_utility.cc
+++ b/modules/rtp_rtcp/source/rtp_utility.cc
@@ -507,6 +507,10 @@
RTC_LOG(WARNING)
<< "RtpGenericFrameDescriptor unsupported by rtp header parser.";
break;
+ case kRtpExtensionHdrMetadata:
+ RTC_LOG(WARNING)
+ << "RtpExtensionHdrMetadata unsupported by rtp header parser.";
+ break;
case kRtpExtensionNone:
case kRtpExtensionNumberOfExtensions: {
RTC_NOTREACHED() << "Invalid extension type: " << type;
diff --git a/test/fuzzers/rtp_packet_fuzzer.cc b/test/fuzzers/rtp_packet_fuzzer.cc
index 8bf8e74..469fb36 100644
--- a/test/fuzzers/rtp_packet_fuzzer.cc
+++ b/test/fuzzers/rtp_packet_fuzzer.cc
@@ -121,6 +121,11 @@
packet.GetExtension<RtpGenericFrameDescriptorExtension>(&descriptor);
break;
}
+ case kRtpExtensionHdrMetadata: {
+ HdrMetadata hdr_metadata;
+ packet.GetExtension<HdrMetadataExtension>(&hdr_metadata);
+ break;
+ }
}
}
}
