Add absolute capture time to video sender path.
Bug: webrtc:10739
Change-Id: I2bbef7275ae065312ad86daaecc773c0ab36a684
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167061
Commit-Queue: Minyue Li <minyue@webrtc.org>
Reviewed-by: Minyue Li <minyue@webrtc.org>
Reviewed-by: Chen Xing <chxg@google.com>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30344}
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc
index eafcdf0..7b7e018 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -22,11 +22,13 @@
#include "api/crypto/frame_encryptor_interface.h"
#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
+#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_format.h"
#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
+#include "modules/rtp_rtcp/source/time_util.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/trace_event.h"
@@ -66,14 +68,16 @@
media_payload.size());
}
-void AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
- const absl::optional<PlayoutDelay>& playout_delay,
- bool set_video_rotation,
- bool set_color_space,
- bool set_frame_marking,
- bool first_packet,
- bool last_packet,
- RtpPacketToSend* packet) {
+void AddRtpHeaderExtensions(
+ const RTPVideoHeader& video_header,
+ const absl::optional<PlayoutDelay>& playout_delay,
+ const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
+ bool set_video_rotation,
+ bool set_color_space,
+ bool set_frame_marking,
+ bool first_packet,
+ bool last_packet,
+ RtpPacketToSend* packet) {
// Color space requires two-byte header extensions if HDR metadata is
// included. Therefore, it's best to add this extension first so that the
// other extensions in the same packet are written as two-byte headers at
@@ -99,6 +103,10 @@
packet->SetExtension<PlayoutDelayLimits>(*playout_delay);
}
+ if (first_packet && absolute_capture_time) {
+ packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time);
+ }
+
if (set_frame_marking) {
FrameMarking frame_marking = video_header.frame_marking;
frame_marking.start_of_frame = first_packet;
@@ -246,7 +254,8 @@
exclude_transport_sequence_number_from_fec_experiment_(
config.field_trials
->Lookup(kExcludeTransportSequenceNumberFromFecFieldTrial)
- .find("Enabled") == 0) {
+ .find("Enabled") == 0),
+ absolute_capture_time_sender_(config.clock) {
RTC_DCHECK(playout_delay_oracle_);
}
@@ -501,21 +510,29 @@
single_packet->SetTimestamp(rtp_timestamp);
single_packet->set_capture_time_ms(capture_time_ms);
+ const absl::optional<AbsoluteCaptureTime> absolute_capture_time =
+ absolute_capture_time_sender_.OnSendPacket(
+ AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
+ single_packet->Csrcs()),
+ single_packet->Timestamp(), kVideoPayloadTypeFrequency,
+ Int64MsToUQ32x32(single_packet->capture_time_ms() + NtpOffsetMs()),
+ /*estimated_capture_clock_offset=*/absl::nullopt);
+
auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
// Simplest way to estimate how much extensions would occupy is to set them.
- AddRtpHeaderExtensions(video_header, playout_delay, set_video_rotation,
- set_color_space, set_frame_marking,
+ AddRtpHeaderExtensions(video_header, playout_delay, absolute_capture_time,
+ set_video_rotation, set_color_space, set_frame_marking,
/*first=*/true, /*last=*/true, single_packet.get());
- AddRtpHeaderExtensions(video_header, playout_delay, set_video_rotation,
- set_color_space, set_frame_marking,
+ AddRtpHeaderExtensions(video_header, playout_delay, absolute_capture_time,
+ set_video_rotation, set_color_space, set_frame_marking,
/*first=*/true, /*last=*/false, first_packet.get());
- AddRtpHeaderExtensions(video_header, playout_delay, set_video_rotation,
- set_color_space, set_frame_marking,
+ AddRtpHeaderExtensions(video_header, playout_delay, absolute_capture_time,
+ set_video_rotation, set_color_space, set_frame_marking,
/*first=*/false, /*last=*/false, middle_packet.get());
- AddRtpHeaderExtensions(video_header, playout_delay, set_video_rotation,
- set_color_space, set_frame_marking,
+ AddRtpHeaderExtensions(video_header, playout_delay, absolute_capture_time,
+ set_video_rotation, set_color_space, set_frame_marking,
/*first=*/false, /*last=*/true, last_packet.get());
RTC_DCHECK_GT(packet_capacity, single_packet->headers_size());
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h
index 9cc7e4f..3f4c676 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video.h
@@ -23,6 +23,7 @@
#include "modules/include/module_common_types.h"
#include "modules/rtp_rtcp/include/flexfec_sender.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
+#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
#include "modules/rtp_rtcp/source/playout_delay_oracle.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
#include "modules/rtp_rtcp/source/rtp_sender.h"
@@ -234,6 +235,8 @@
const bool generic_descriptor_auth_experiment_;
const bool exclude_transport_sequence_number_from_fec_experiment_;
+
+ AbsoluteCaptureTimeSender absolute_capture_time_sender_;
};
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
index 1663ad5..7ccd0ac 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
@@ -24,6 +24,7 @@
#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
+#include "modules/rtp_rtcp/source/time_util.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/rate_limiter.h"
#include "test/gmock.h"
@@ -44,6 +45,7 @@
kTransportSequenceNumberExtensionId,
kVideoRotationExtensionId,
kVideoTimingExtensionId,
+ kAbsoluteCaptureTimeExtensionId,
};
constexpr int kPayload = 100;
@@ -73,6 +75,8 @@
kGenericDescriptorId01);
receivers_extensions_.Register<FrameMarkingExtension>(
kFrameMarkingExtensionId);
+ receivers_extensions_.Register<AbsoluteCaptureTimeExtension>(
+ kAbsoluteCaptureTimeExtensionId);
}
bool SendRtp(const uint8_t* data,
@@ -85,6 +89,9 @@
bool SendRtcp(const uint8_t* data, size_t len) override { return false; }
const RtpPacketReceived& last_sent_packet() { return sent_packets_.back(); }
int packets_sent() { return sent_packets_.size(); }
+ const std::vector<RtpPacketReceived>& sent_packets() const {
+ return sent_packets_;
+ }
private:
RtpHeaderExtensionMap receivers_extensions_;
@@ -606,6 +613,33 @@
UsesMinimalVp8DescriptorWhenGenericFrameDescriptorExtensionIsUsed(1);
}
+TEST_P(RtpSenderVideoTest, AbsoluteCaptureTime) {
+ constexpr int64_t kAbsoluteCaptureTimestampMs = 12345678;
+ uint8_t kFrame[kMaxPacketLength];
+ rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::kUri,
+ kAbsoluteCaptureTimeExtensionId);
+
+ RTPVideoHeader hdr;
+ hdr.frame_type = VideoFrameType::kVideoFrameKey;
+ rtp_sender_video_.SendVideo(kPayload, kType, kTimestamp,
+ kAbsoluteCaptureTimestampMs, kFrame, nullptr, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+
+ // It is expected that one and only one of the packets sent on this video
+ // frame has absolute capture time header extension.
+ int packets_with_abs_capture_time = 0;
+ for (const RtpPacketReceived& packet : transport_.sent_packets()) {
+ auto absolute_capture_time =
+ packet.GetExtension<AbsoluteCaptureTimeExtension>();
+ if (absolute_capture_time) {
+ ++packets_with_abs_capture_time;
+ EXPECT_EQ(absolute_capture_time->absolute_capture_timestamp,
+ Int64MsToUQ32x32(kAbsoluteCaptureTimestampMs + NtpOffsetMs()));
+ }
+ }
+ EXPECT_EQ(packets_with_abs_capture_time, 1);
+}
+
INSTANTIATE_TEST_SUITE_P(WithAndWithoutOverhead,
RtpSenderVideoTest,
::testing::Bool());
