Surface `local_capture_clock_offset` from `RtpSource`
- Propagating `RtpPacketInfo::local_capture_clock_offset`, an
existing field that is related to the abs-capture-timestamp
header extension field `estimated_capture_clock_offset`
- Propagated through `SourceTracker::SourceEntry`
Bug: webrtc:10739, b/246753278
Change-Id: I21d9841e4f3a35da5f8d7b31582898309421d524
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/275241
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38129}
diff --git a/api/BUILD.gn b/api/BUILD.gn
index e9db503..10337cb 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -94,6 +94,7 @@
":rtp_headers",
":scoped_refptr",
"../rtc_base/system:rtc_export",
+ "units:time_delta",
"units:timestamp",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
diff --git a/api/rtp_packet_info.h b/api/rtp_packet_info.h
index f9980a1c..bc8784c 100644
--- a/api/rtp_packet_info.h
+++ b/api/rtp_packet_info.h
@@ -17,6 +17,7 @@
#include "absl/types/optional.h"
#include "api/rtp_headers.h"
+#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "rtc_base/system/rtc_export.h"
@@ -92,11 +93,11 @@
return *this;
}
- const absl::optional<int64_t>& local_capture_clock_offset() const {
+ const absl::optional<TimeDelta>& local_capture_clock_offset() const {
return local_capture_clock_offset_;
}
RtpPacketInfo& set_local_capture_clock_offset(
- const absl::optional<int64_t>& value) {
+ absl::optional<TimeDelta> value) {
local_capture_clock_offset_ = value;
return *this;
}
@@ -117,16 +118,14 @@
// Fields from the Absolute Capture Time header extension:
// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time
- // To not be confused with `local_capture_clock_offset_`, the
- // `estimated_capture_clock_offset` in `absolute_capture_time_` should
- // represent the clock offset between a remote sender and the capturer, and
- // thus equals to the corresponding values in the received RTP packets,
- // subjected to possible interpolations.
absl::optional<AbsoluteCaptureTime> absolute_capture_time_;
- // Clock offset against capturer's clock. Should be derived from the estimated
- // capture clock offset defined in the Absolute Capture Time header extension.
- absl::optional<int64_t> local_capture_clock_offset_;
+ // Clock offset between the local clock and the capturer's clock.
+ // Do not confuse with `AbsoluteCaptureTime::estimated_capture_clock_offset`
+ // which instead represents the clock offset between a remote sender and the
+ // capturer. The following holds:
+ // Capture's NTP Clock = Local NTP Clock + Local-Capture Clock Offset
+ absl::optional<TimeDelta> local_capture_clock_offset_;
};
bool operator==(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs);
diff --git a/api/rtp_packet_info_unittest.cc b/api/rtp_packet_info_unittest.cc
index 80abccc..d35edf7 100644
--- a/api/rtp_packet_info_unittest.cc
+++ b/api/rtp_packet_info_unittest.cc
@@ -9,6 +9,7 @@
*/
#include "api/rtp_packet_infos.h"
+#include "api/units/time_delta.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -186,7 +187,7 @@
}
TEST(RtpPacketInfoTest, LocalCaptureClockOffset) {
- constexpr absl::optional<int64_t> kValue = 10;
+ constexpr TimeDelta kValue = TimeDelta::Micros(8868963877546349045LL);
RtpPacketInfo lhs;
RtpPacketInfo rhs;
diff --git a/api/transport/rtp/BUILD.gn b/api/transport/rtp/BUILD.gn
index 26036c7..205bbcc 100644
--- a/api/transport/rtp/BUILD.gn
+++ b/api/transport/rtp/BUILD.gn
@@ -13,6 +13,7 @@
sources = [ "rtp_source.h" ]
deps = [
"../../../api:rtp_headers",
+ "../../../api/units:time_delta",
"../../../rtc_base:checks",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
diff --git a/api/transport/rtp/rtp_source.h b/api/transport/rtp/rtp_source.h
index c19cfeb..e51dcd7 100644
--- a/api/transport/rtp/rtp_source.h
+++ b/api/transport/rtp/rtp_source.h
@@ -15,6 +15,7 @@
#include "absl/types/optional.h"
#include "api/rtp_headers.h"
+#include "api/units/time_delta.h"
#include "rtc_base/checks.h"
namespace webrtc {
@@ -28,7 +29,17 @@
public:
struct Extensions {
absl::optional<uint8_t> audio_level;
+
+ // Fields from the Absolute Capture Time header extension:
+ // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
+
+ // Clock offset between the local clock and the capturer's clock.
+ // Do not confuse with `AbsoluteCaptureTime::estimated_capture_clock_offset`
+ // which instead represents the clock offset between a remote sender and the
+ // capturer. The following holds:
+ // Capture's NTP Clock = Local NTP Clock + Local-Capture Clock Offset
+ absl::optional<TimeDelta> local_capture_clock_offset;
};
RtpSource() = delete;
@@ -74,6 +85,10 @@
return extensions_.absolute_capture_time;
}
+ absl::optional<TimeDelta> local_capture_clock_offset() const {
+ return extensions_.local_capture_clock_offset;
+ }
+
bool operator==(const RtpSource& o) const {
return timestamp_ms_ == o.timestamp_ms() && source_id_ == o.source_id() &&
source_type_ == o.source_type() &&
diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc
index 5f54c01..91294ad 100644
--- a/audio/channel_receive.cc
+++ b/audio/channel_receive.cc
@@ -23,6 +23,7 @@
#include "api/sequence_checker.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/task_queue/task_queue_base.h"
+#include "api/units/time_delta.h"
#include "audio/audio_level.h"
#include "audio/channel_receive_frame_transformer_delegate.h"
#include "audio/channel_send.h"
@@ -48,6 +49,7 @@
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/metrics.h"
+#include "system_wrappers/include/ntp_time.h"
namespace webrtc {
namespace voe {
@@ -464,14 +466,19 @@
// Fill in local capture clock offset in `audio_frame->packet_infos_`.
RtpPacketInfos::vector_type packet_infos;
for (auto& packet_info : audio_frame->packet_infos_) {
- absl::optional<int64_t> local_capture_clock_offset;
+ absl::optional<int64_t> local_capture_clock_offset_q32x32;
if (packet_info.absolute_capture_time().has_value()) {
- local_capture_clock_offset =
+ local_capture_clock_offset_q32x32 =
capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset(
packet_info.absolute_capture_time()
->estimated_capture_clock_offset);
}
RtpPacketInfo new_packet_info(packet_info);
+ absl::optional<TimeDelta> local_capture_clock_offset;
+ if (local_capture_clock_offset_q32x32.has_value()) {
+ local_capture_clock_offset = TimeDelta::Millis(
+ UQ32x32ToInt64Ms(*local_capture_clock_offset_q32x32));
+ }
new_packet_info.set_local_capture_clock_offset(local_capture_clock_offset);
packet_infos.push_back(std::move(new_packet_info));
}
diff --git a/modules/rtp_rtcp/source/source_tracker.cc b/modules/rtp_rtcp/source/source_tracker.cc
index f9aa003..090e435 100644
--- a/modules/rtp_rtcp/source/source_tracker.cc
+++ b/modules/rtp_rtcp/source/source_tracker.cc
@@ -27,7 +27,7 @@
int64_t now_ms = clock_->TimeInMilliseconds();
MutexLock lock_scope(&lock_);
- for (const auto& packet_info : packet_infos) {
+ for (const RtpPacketInfo& packet_info : packet_infos) {
for (uint32_t csrc : packet_info.csrcs()) {
SourceKey key(RtpSourceType::CSRC, csrc);
SourceEntry& entry = UpdateEntry(key);
@@ -35,6 +35,8 @@
entry.timestamp_ms = now_ms;
entry.audio_level = packet_info.audio_level();
entry.absolute_capture_time = packet_info.absolute_capture_time();
+ entry.local_capture_clock_offset =
+ packet_info.local_capture_clock_offset();
entry.rtp_timestamp = packet_info.rtp_timestamp();
}
@@ -44,6 +46,7 @@
entry.timestamp_ms = now_ms;
entry.audio_level = packet_info.audio_level();
entry.absolute_capture_time = packet_info.absolute_capture_time();
+ entry.local_capture_clock_offset = packet_info.local_capture_clock_offset();
entry.rtp_timestamp = packet_info.rtp_timestamp();
}
@@ -64,7 +67,10 @@
sources.emplace_back(
entry.timestamp_ms, key.source, key.source_type, entry.rtp_timestamp,
- RtpSource::Extensions{entry.audio_level, entry.absolute_capture_time});
+ RtpSource::Extensions{
+ .audio_level = entry.audio_level,
+ .absolute_capture_time = entry.absolute_capture_time,
+ .local_capture_clock_offset = entry.local_capture_clock_offset});
}
return sources;
diff --git a/modules/rtp_rtcp/source/source_tracker.h b/modules/rtp_rtcp/source/source_tracker.h
index 3f3ef8c..f9e8354 100644
--- a/modules/rtp_rtcp/source/source_tracker.h
+++ b/modules/rtp_rtcp/source/source_tracker.h
@@ -20,6 +20,7 @@
#include "absl/types/optional.h"
#include "api/rtp_packet_infos.h"
#include "api/transport/rtp/rtp_source.h"
+#include "api/units/time_delta.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/clock.h"
@@ -95,6 +96,13 @@
// https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
+ // Clock offset between the local clock and the capturer's clock.
+ // Do not confuse with `AbsoluteCaptureTime::estimated_capture_clock_offset`
+ // which instead represents the clock offset between a remote sender and the
+ // capturer. The following holds:
+ // Capture's NTP Clock = Local NTP Clock + Local-Capture Clock Offset
+ absl::optional<TimeDelta> local_capture_clock_offset;
+
// RTP timestamp of the most recent packet used to assemble the frame
// associated with `timestamp_ms`.
uint32_t rtp_timestamp;
diff --git a/modules/rtp_rtcp/source/source_tracker_unittest.cc b/modules/rtp_rtcp/source/source_tracker_unittest.cc
index e0440a0..b3e3102 100644
--- a/modules/rtp_rtcp/source/source_tracker_unittest.cc
+++ b/modules/rtp_rtcp/source/source_tracker_unittest.cc
@@ -22,6 +22,7 @@
#include "api/rtp_headers.h"
#include "api/rtp_packet_info.h"
#include "api/rtp_packet_infos.h"
+#include "system_wrappers/include/ntp_time.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -47,8 +48,9 @@
const int64_t now_ms = clock_->TimeInMilliseconds();
for (const auto& packet_info : packet_infos) {
- RtpSource::Extensions extensions = {packet_info.audio_level(),
- packet_info.absolute_capture_time()};
+ RtpSource::Extensions extensions = {
+ packet_info.audio_level(), packet_info.absolute_capture_time(),
+ packet_info.local_capture_clock_offset()};
for (const auto& csrc : packet_info.csrcs()) {
entries_.emplace_front(now_ms, csrc, RtpSourceType::CSRC,
@@ -112,7 +114,8 @@
.emplace_back(GenerateSsrc(), GenerateCsrcs(), GenerateRtpTimestamp(),
GenerateReceiveTime())
.set_audio_level(GenerateAudioLevel())
- .set_absolute_capture_time(GenerateAbsoluteCaptureTime());
+ .set_absolute_capture_time(GenerateAbsoluteCaptureTime())
+ .set_local_capture_clock_offset(GenerateLocalCaptureClockOffset());
}
return RtpPacketInfos(std::move(packet_infos));
@@ -193,6 +196,14 @@
return value;
}
+ absl::optional<TimeDelta> GenerateLocalCaptureClockOffset() {
+ if (std::bernoulli_distribution(0.5)(generator_)) {
+ return absl::nullopt;
+ }
+ return TimeDelta::Millis(
+ UQ32x32ToInt64Ms(std::uniform_int_distribution<int64_t>()(generator_)));
+ }
+
Timestamp GenerateReceiveTime() {
return Timestamp::Micros(
std::uniform_int_distribution<int64_t>()(generator_));
@@ -254,6 +265,7 @@
constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime =
AbsoluteCaptureTime{/*absolute_capture_timestamp=*/12,
/*estimated_capture_clock_offset=*/absl::nullopt};
+ constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset = absl::nullopt;
constexpr Timestamp kReceiveTime0 = Timestamp::Millis(60);
constexpr Timestamp kReceiveTime1 = Timestamp::Millis(70);
@@ -263,18 +275,22 @@
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc1, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kReceiveTime0)
.set_audio_level(kAudioLevel0)
- .set_absolute_capture_time(kAbsoluteCaptureTime),
+ .set_absolute_capture_time(kAbsoluteCaptureTime)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset),
RtpPacketInfo(kSsrc2, {kCsrcs2}, kRtpTimestamp1, kReceiveTime1)
.set_audio_level(kAudioLevel1)
- .set_absolute_capture_time(kAbsoluteCaptureTime)}));
+ .set_absolute_capture_time(kAbsoluteCaptureTime)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset)}));
int64_t timestamp_ms = clock.TimeInMilliseconds();
constexpr RtpSource::Extensions extensions0 = {
.audio_level = kAudioLevel0,
- .absolute_capture_time = kAbsoluteCaptureTime};
+ .absolute_capture_time = kAbsoluteCaptureTime,
+ .local_capture_clock_offset = kLocalCaptureClockOffset};
constexpr RtpSource::Extensions extensions1 = {
.audio_level = kAudioLevel1,
- .absolute_capture_time = kAbsoluteCaptureTime};
+ .absolute_capture_time = kAbsoluteCaptureTime,
+ .local_capture_clock_offset = kLocalCaptureClockOffset};
EXPECT_THAT(tracker.GetSources(),
ElementsAre(RtpSource(timestamp_ms, kSsrc2, RtpSourceType::SSRC,
@@ -303,6 +319,7 @@
constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime =
AbsoluteCaptureTime{/*absolute_capture_timestamp=*/12,
/*estimated_capture_clock_offset=*/absl::nullopt};
+ constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset = absl::nullopt;
constexpr Timestamp kReceiveTime0 = Timestamp::Millis(60);
constexpr Timestamp kReceiveTime1 = Timestamp::Millis(70);
constexpr Timestamp kReceiveTime2 = Timestamp::Millis(80);
@@ -313,22 +330,31 @@
tracker.OnFrameDelivered(RtpPacketInfos({
RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kReceiveTime0)
.set_audio_level(kAudioLevel0)
- .set_absolute_capture_time(kAbsoluteCaptureTime),
+ .set_absolute_capture_time(kAbsoluteCaptureTime)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset),
RtpPacketInfo(kSsrc, {kCsrcs2}, kRtpTimestamp1, kReceiveTime1)
.set_audio_level(kAudioLevel1)
- .set_absolute_capture_time(kAbsoluteCaptureTime),
+ .set_absolute_capture_time(kAbsoluteCaptureTime)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset),
RtpPacketInfo(kSsrc, {kCsrcs0}, kRtpTimestamp2, kReceiveTime2)
.set_audio_level(kAudioLevel2)
- .set_absolute_capture_time(kAbsoluteCaptureTime),
+ .set_absolute_capture_time(kAbsoluteCaptureTime)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset),
}));
int64_t timestamp_ms = clock.TimeInMilliseconds();
- constexpr RtpSource::Extensions extensions0 = {kAudioLevel0,
- kAbsoluteCaptureTime};
- constexpr RtpSource::Extensions extensions1 = {kAudioLevel1,
- kAbsoluteCaptureTime};
- constexpr RtpSource::Extensions extensions2 = {kAudioLevel2,
- kAbsoluteCaptureTime};
+ constexpr RtpSource::Extensions extensions0 = {
+ .audio_level = kAudioLevel0,
+ .absolute_capture_time = kAbsoluteCaptureTime,
+ .local_capture_clock_offset = kLocalCaptureClockOffset};
+ constexpr RtpSource::Extensions extensions1 = {
+ .audio_level = kAudioLevel1,
+ .absolute_capture_time = kAbsoluteCaptureTime,
+ .local_capture_clock_offset = kLocalCaptureClockOffset};
+ constexpr RtpSource::Extensions extensions2 = {
+ .audio_level = kAudioLevel2,
+ .absolute_capture_time = kAbsoluteCaptureTime,
+ .local_capture_clock_offset = kLocalCaptureClockOffset};
EXPECT_THAT(tracker.GetSources(),
ElementsAre(RtpSource(timestamp_ms, kSsrc, RtpSourceType::SSRC,
@@ -359,19 +385,28 @@
AbsoluteCaptureTime{56, 78};
constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime2 =
AbsoluteCaptureTime{89, 90};
+ constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset0 =
+ TimeDelta::Millis(123);
+ constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset1 =
+ TimeDelta::Millis(456);
+ constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset2 =
+ TimeDelta::Millis(789);
constexpr Timestamp kReceiveTime0 = Timestamp::Millis(60);
constexpr Timestamp kReceiveTime1 = Timestamp::Millis(61);
constexpr Timestamp kReceiveTime2 = Timestamp::Millis(62);
constexpr RtpSource::Extensions extensions0 = {
.audio_level = kAudioLevel0,
- .absolute_capture_time = kAbsoluteCaptureTime0};
+ .absolute_capture_time = kAbsoluteCaptureTime0,
+ .local_capture_clock_offset = kLocalCaptureClockOffset0};
constexpr RtpSource::Extensions extensions1 = {
.audio_level = kAudioLevel1,
- .absolute_capture_time = kAbsoluteCaptureTime1};
+ .absolute_capture_time = kAbsoluteCaptureTime1,
+ .local_capture_clock_offset = kLocalCaptureClockOffset1};
constexpr RtpSource::Extensions extensions2 = {
.audio_level = kAudioLevel2,
- .absolute_capture_time = kAbsoluteCaptureTime2};
+ .absolute_capture_time = kAbsoluteCaptureTime2,
+ .local_capture_clock_offset = kLocalCaptureClockOffset2};
SimulatedClock clock(1000000000000ULL);
SourceTracker tracker(&clock);
@@ -379,7 +414,8 @@
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc1, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kReceiveTime0)
.set_audio_level(kAudioLevel0)
- .set_absolute_capture_time(kAbsoluteCaptureTime0)}));
+ .set_absolute_capture_time(kAbsoluteCaptureTime0)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset0)}));
int64_t timestamp_ms_0 = clock.TimeInMilliseconds();
EXPECT_THAT(
@@ -397,7 +433,8 @@
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc1, {kCsrcs0, kCsrcs2}, kRtpTimestamp1, kReceiveTime1)
.set_audio_level(kAudioLevel1)
- .set_absolute_capture_time(kAbsoluteCaptureTime1)}));
+ .set_absolute_capture_time(kAbsoluteCaptureTime1)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset1)}));
int64_t timestamp_ms_1 = clock.TimeInMilliseconds();
@@ -418,7 +455,8 @@
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc2, {kCsrcs0}, kRtpTimestamp2, kReceiveTime2)
.set_audio_level(kAudioLevel2)
- .set_absolute_capture_time(kAbsoluteCaptureTime2)}));
+ .set_absolute_capture_time(kAbsoluteCaptureTime2)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset2)}));
int64_t timestamp_ms_2 = clock.TimeInMilliseconds();
@@ -449,6 +487,10 @@
AbsoluteCaptureTime{12, 34};
constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime1 =
AbsoluteCaptureTime{56, 78};
+ constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset0 =
+ TimeDelta::Millis(123);
+ constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset1 =
+ TimeDelta::Millis(456);
constexpr Timestamp kReceiveTime0 = Timestamp::Millis(60);
constexpr Timestamp kReceiveTime1 = Timestamp::Millis(61);
@@ -458,14 +500,16 @@
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kReceiveTime0)
.set_audio_level(kAudioLevel0)
- .set_absolute_capture_time(kAbsoluteCaptureTime0)}));
+ .set_absolute_capture_time(kAbsoluteCaptureTime0)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset0)}));
clock.AdvanceTimeMilliseconds(17);
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs2}, kRtpTimestamp1, kReceiveTime1)
.set_audio_level(kAudioLevel1)
- .set_absolute_capture_time(kAbsoluteCaptureTime1)}));
+ .set_absolute_capture_time(kAbsoluteCaptureTime1)
+ .set_local_capture_clock_offset(kLocalCaptureClockOffset1)}));
int64_t timestamp_ms_1 = clock.TimeInMilliseconds();
@@ -473,7 +517,8 @@
constexpr RtpSource::Extensions extensions1 = {
.audio_level = kAudioLevel1,
- .absolute_capture_time = kAbsoluteCaptureTime1};
+ .absolute_capture_time = kAbsoluteCaptureTime1,
+ .local_capture_clock_offset = kLocalCaptureClockOffset1};
EXPECT_THAT(
tracker.GetSources(),
