Allow providing the absolute capture time extension when packetizing a frame.
Bug: b/150859541
Change-Id: Iffb6ee84f49ffa64fdb0633248864d2dfd6e9ff3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/234868
Commit-Queue: Paul Hallak <phallak@google.com>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35194}
diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn
index 00431dc..f7a0506 100644
--- a/modules/rtp_rtcp/BUILD.gn
+++ b/modules/rtp_rtcp/BUILD.gn
@@ -398,6 +398,7 @@
"source/rtp_video_header.h",
]
deps = [
+ "../../api:rtp_headers",
"../../api/transport/rtp:dependency_descriptor",
"../../api/video:video_frame",
"../../api/video:video_frame_type",
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc
index 9e909c9..c9fc782 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -315,12 +315,10 @@
allocation_ = std::move(allocation);
}
-void RTPSenderVideo::AddRtpHeaderExtensions(
- const RTPVideoHeader& video_header,
- const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
- bool first_packet,
- bool last_packet,
- RtpPacketToSend* packet) const {
+void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
+ bool first_packet,
+ bool last_packet,
+ RtpPacketToSend* packet) const {
// Send color space when changed or if the frame is a key frame. Keep
// sending color space information until the first base layer frame to
// guarantee that the information is retrieved by the receiver.
@@ -369,8 +367,9 @@
packet->SetExtension<PlayoutDelayLimits>(current_playout_delay_);
}
- if (first_packet && absolute_capture_time) {
- packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time);
+ if (first_packet && video_header.absolute_capture_time.has_value()) {
+ packet->SetExtension<AbsoluteCaptureTimeExtension>(
+ *video_header.absolute_capture_time);
}
if (video_header.generic) {
@@ -477,8 +476,7 @@
int64_t capture_time_ms,
rtc::ArrayView<const uint8_t> payload,
RTPVideoHeader video_header,
- absl::optional<int64_t> expected_retransmission_time_ms,
- absl::optional<int64_t> estimated_capture_clock_offset_ms) {
+ absl::optional<int64_t> expected_retransmission_time_ms) {
#if RTC_TRACE_EVENTS_ENABLED
TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type",
FrameTypeToString(video_header.frame_type));
@@ -538,22 +536,31 @@
single_packet->SetTimestamp(rtp_timestamp);
single_packet->set_capture_time_ms(capture_time_ms);
- const absl::optional<AbsoluteCaptureTime> absolute_capture_time =
+ // Construct the absolute capture time extension if not provided.
+ if (!video_header.absolute_capture_time.has_value()) {
+ video_header.absolute_capture_time.emplace();
+ video_header.absolute_capture_time->absolute_capture_timestamp =
+ Int64MsToUQ32x32(
+ clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms));
+ if (include_capture_clock_offset_) {
+ video_header.absolute_capture_time->estimated_capture_clock_offset = 0;
+ }
+ }
+
+ // Let `absolute_capture_time_sender_` decide if the extension should be sent.
+ video_header.absolute_capture_time =
absolute_capture_time_sender_.OnSendPacket(
AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
single_packet->Csrcs()),
single_packet->Timestamp(), kVideoPayloadTypeFrequency,
- Int64MsToUQ32x32(
- clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms)),
- /*estimated_capture_clock_offset=*/
- include_capture_clock_offset_ ? estimated_capture_clock_offset_ms
- : absl::nullopt);
+ video_header.absolute_capture_time->absolute_capture_timestamp,
+ video_header.absolute_capture_time->estimated_capture_clock_offset);
auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
// Simplest way to estimate how much extensions would occupy is to set them.
- AddRtpHeaderExtensions(video_header, absolute_capture_time,
+ AddRtpHeaderExtensions(video_header,
/*first_packet=*/true, /*last_packet=*/true,
single_packet.get());
if (video_structure_ != nullptr &&
@@ -568,13 +575,13 @@
video_structure_ = nullptr;
}
- AddRtpHeaderExtensions(video_header, absolute_capture_time,
+ AddRtpHeaderExtensions(video_header,
/*first_packet=*/true, /*last_packet=*/false,
first_packet.get());
- AddRtpHeaderExtensions(video_header, absolute_capture_time,
+ AddRtpHeaderExtensions(video_header,
/*first_packet=*/false, /*last_packet=*/false,
middle_packet.get());
- AddRtpHeaderExtensions(video_header, absolute_capture_time,
+ AddRtpHeaderExtensions(video_header,
/*first_packet=*/false, /*last_packet=*/true,
last_packet.get());
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h
index f4f15f3..5164969 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video.h
@@ -90,20 +90,14 @@
// expected_retransmission_time_ms.has_value() -> retransmission allowed.
// `capture_time_ms` and `clock::CurrentTime` should be using the same epoch.
- // Calls to this method is assumed to be externally serialized.
- // `estimated_capture_clock_offset_ms` is an estimated clock offset between
- // this sender and the original capturer, for this video packet. See
- // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time for more
- // details. If the sender and the capture has the same clock, it is supposed
- // to be zero valued, which is given as the default.
+ // Calls to this method are assumed to be externally serialized.
bool SendVideo(int payload_type,
absl::optional<VideoCodecType> codec_type,
uint32_t rtp_timestamp,
int64_t capture_time_ms,
rtc::ArrayView<const uint8_t> payload,
RTPVideoHeader video_header,
- absl::optional<int64_t> expected_retransmission_time_ms,
- absl::optional<int64_t> estimated_capture_clock_offset_ms = 0);
+ absl::optional<int64_t> expected_retransmission_time_ms);
bool SendEncodedImage(
int payload_type,
@@ -170,12 +164,10 @@
const FrameDependencyStructure* video_structure);
void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation);
- void AddRtpHeaderExtensions(
- const RTPVideoHeader& video_header,
- const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
- bool first_packet,
- bool last_packet,
- RtpPacketToSend* packet) const
+ void AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
+ bool first_packet,
+ bool last_packet,
+ RtpPacketToSend* packet) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
index d3782be..d746ad1 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
@@ -16,6 +16,7 @@
#include <vector>
#include "absl/memory/memory.h"
+#include "api/rtp_headers.h"
#include "api/test/mock_frame_encryptor.h"
#include "api/transport/field_trial_based_config.h"
#include "api/transport/rtp/dependency_descriptor.h"
@@ -1181,24 +1182,27 @@
kAbsoluteCaptureTimestampMs, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
+ absl::optional<AbsoluteCaptureTime> absolute_capture_time;
+
// It is expected that one and only one of the packets sent on this video
- // frame has absolute capture time header extension. And no absolute capture
- // time header extensions include capture clock offset.
- int packets_with_abs_capture_time = 0;
+ // frame has absolute capture time header extension.
for (const RtpPacketReceived& packet : transport_.sent_packets()) {
- auto absolute_capture_time =
- packet.GetExtension<AbsoluteCaptureTimeExtension>();
- if (absolute_capture_time) {
- ++packets_with_abs_capture_time;
- EXPECT_EQ(
- absolute_capture_time->absolute_capture_timestamp,
- Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
- kAbsoluteCaptureTimestampMs)));
- EXPECT_FALSE(
- absolute_capture_time->estimated_capture_clock_offset.has_value());
+ if (absolute_capture_time.has_value()) {
+ EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
+ } else {
+ absolute_capture_time =
+ packet.GetExtension<AbsoluteCaptureTimeExtension>();
}
}
- EXPECT_EQ(packets_with_abs_capture_time, 1);
+
+ // Verify the capture timestamp and that the clock offset is not set.
+ ASSERT_TRUE(absolute_capture_time.has_value());
+ EXPECT_EQ(
+ absolute_capture_time->absolute_capture_timestamp,
+ Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
+ kAbsoluteCaptureTimestampMs)));
+ EXPECT_FALSE(
+ absolute_capture_time->estimated_capture_clock_offset.has_value());
}
// Essentially the same test as AbsoluteCaptureTime but with a field trial.
@@ -1214,31 +1218,64 @@
kAbsoluteCaptureTimeExtensionId);
RTPVideoHeader hdr;
- const absl::optional<int64_t> kExpectedCaptureClockOffset =
- absl::make_optional(1234);
hdr.frame_type = VideoFrameType::kVideoFrameKey;
- rtp_sender_video_->SendVideo(
- kPayload, kType, kTimestamp, kAbsoluteCaptureTimestampMs, kFrame, hdr,
- kDefaultExpectedRetransmissionTimeMs, kExpectedCaptureClockOffset);
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
+ kAbsoluteCaptureTimestampMs, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+
+ absl::optional<AbsoluteCaptureTime> absolute_capture_time;
// It is expected that one and only one of the packets sent on this video
- // frame has absolute capture time header extension. And it includes capture
- // clock offset.
- int packets_with_abs_capture_time = 0;
+ // frame has absolute capture time header extension.
for (const RtpPacketReceived& packet : transport_.sent_packets()) {
- auto absolute_capture_time =
- packet.GetExtension<AbsoluteCaptureTimeExtension>();
- if (absolute_capture_time) {
- ++packets_with_abs_capture_time;
- EXPECT_EQ(
- absolute_capture_time->absolute_capture_timestamp,
- Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
- kAbsoluteCaptureTimestampMs)));
- EXPECT_EQ(kExpectedCaptureClockOffset,
- absolute_capture_time->estimated_capture_clock_offset);
+ if (absolute_capture_time.has_value()) {
+ EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
+ } else {
+ absolute_capture_time =
+ packet.GetExtension<AbsoluteCaptureTimeExtension>();
}
}
- EXPECT_EQ(packets_with_abs_capture_time, 1);
+
+ // Verify the capture timestamp and that the clock offset is set to zero.
+ ASSERT_TRUE(absolute_capture_time.has_value());
+ EXPECT_EQ(
+ absolute_capture_time->absolute_capture_timestamp,
+ Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
+ kAbsoluteCaptureTimestampMs)));
+ EXPECT_EQ(absolute_capture_time->estimated_capture_clock_offset, 0);
+}
+
+TEST_P(RtpSenderVideoTest, AbsoluteCaptureTimeWithExtensionProvided) {
+ constexpr AbsoluteCaptureTime kAbsoluteCaptureTime = {
+ 123,
+ absl::optional<int64_t>(456),
+ };
+ uint8_t kFrame[kMaxPacketLength];
+ rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::Uri(),
+ kAbsoluteCaptureTimeExtensionId);
+
+ RTPVideoHeader hdr;
+ hdr.frame_type = VideoFrameType::kVideoFrameKey;
+ hdr.absolute_capture_time = kAbsoluteCaptureTime;
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
+ /*capture_time_ms=*/789, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+
+ absl::optional<AbsoluteCaptureTime> absolute_capture_time;
+
+ // It is expected that one and only one of the packets sent on this video
+ // frame has absolute capture time header extension.
+ for (const RtpPacketReceived& packet : transport_.sent_packets()) {
+ if (absolute_capture_time.has_value()) {
+ EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
+ } else {
+ absolute_capture_time =
+ packet.GetExtension<AbsoluteCaptureTimeExtension>();
+ }
+ }
+
+ // Verify the extension.
+ EXPECT_EQ(absolute_capture_time, kAbsoluteCaptureTime);
}
TEST_P(RtpSenderVideoTest, PopulatesPlayoutDelay) {
diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h
index c1be76f..115b17d 100644
--- a/modules/rtp_rtcp/source/rtp_video_header.h
+++ b/modules/rtp_rtcp/source/rtp_video_header.h
@@ -16,6 +16,7 @@
#include "absl/container/inlined_vector.h"
#include "absl/types/optional.h"
#include "absl/types/variant.h"
+#include "api/rtp_headers.h"
#include "api/transport/rtp/dependency_descriptor.h"
#include "api/video/color_space.h"
#include "api/video/video_codec_type.h"
@@ -81,6 +82,11 @@
// carries the webrtc::VideoFrame id field from the sender to the receiver.
absl::optional<uint16_t> video_frame_tracking_id;
RTPVideoTypeHeader video_type_header;
+
+ // When provided, is sent as is as an RTP header extension according to
+ // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time.
+ // Otherwise, it is derived from other relevant information.
+ absl::optional<AbsoluteCaptureTime> absolute_capture_time;
};
} // namespace webrtc
