Add support for the Absolute Capture Timestamp extension to TransformableAudioFrameInterface
Bug: chromium:391114797
Change-Id: Iad09ed3b509ce7874c44cd17c1e87b6945b14b07
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/377121
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Jakob Ivarsson‎ <jakobi@webrtc.org>
Commit-Queue: Guido Urdaneta <guidou@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43893}
diff --git a/api/BUILD.gn b/api/BUILD.gn
index 010695a..e2d2a3b 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -410,6 +410,7 @@
":scoped_refptr",
"../rtc_base:refcount",
"../rtc_base/system:rtc_export",
+ "units:time_delta",
"units:timestamp",
"video:encoded_frame",
"video:video_frame_metadata",
diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h
index 7133efa..63df71c 100644
--- a/api/frame_transformer_interface.h
+++ b/api/frame_transformer_interface.h
@@ -19,6 +19,7 @@
#include "api/array_view.h"
#include "api/ref_count.h"
#include "api/scoped_refptr.h"
+#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "api/video/video_frame_metadata.h"
#include "rtc_base/system/rtc_export.h"
@@ -99,6 +100,7 @@
virtual const std::optional<uint16_t> SequenceNumber() const = 0;
+ // TODO(crbug.com/391114797): Delete this function.
virtual std::optional<uint64_t> AbsoluteCaptureTimestamp() const = 0;
enum class FrameType { kEmptyFrame, kAudioFrameSpeech, kAudioFrameCN };
@@ -115,6 +117,21 @@
// Timestamp at which the packet has been first seen on the network interface.
// Only defined for received audio packet.
virtual std::optional<Timestamp> ReceiveTime() const = 0;
+
+ // Timestamp at which the frame was captured in the capturer system.
+ // The timestamp is expressed in the capturer system's clock relative to the
+ // NTP epoch (January 1st 1970 00:00 UTC)
+ // Accessible only if the absolute capture timestamp header extension is
+ // enabled.
+ virtual std::optional<Timestamp> CaptureTime() const = 0;
+
+ // Offset between the sender system's clock and the capturer system's clock.
+ // Can be used to express the capture time in the local system's clock as
+ // long as the local system can determine the offset between its local clock
+ // and the sender system's clock.
+ // Accessible only if the absolute capture timestamp header extension is
+ // enabled.
+ virtual std::optional<TimeDelta> SenderCaptureTimeOffset() const = 0;
};
// Objects implement this interface to be notified with the transformed frame.
diff --git a/api/test/mock_transformable_audio_frame.h b/api/test/mock_transformable_audio_frame.h
index d2100e6..762a9dc 100644
--- a/api/test/mock_transformable_audio_frame.h
+++ b/api/test/mock_transformable_audio_frame.h
@@ -56,6 +56,11 @@
MOCK_METHOD(std::optional<uint8_t>, AudioLevel, (), (const, override));
MOCK_METHOD(std::optional<Timestamp>, ReceiveTime, (), (const, override));
+ MOCK_METHOD(std::optional<Timestamp>, CaptureTime, (), (const, override));
+ MOCK_METHOD(std::optional<TimeDelta>,
+ SenderCaptureTimeOffset,
+ (),
+ (const, override));
};
} // namespace webrtc
diff --git a/audio/channel_receive_frame_transformer_delegate.cc b/audio/channel_receive_frame_transformer_delegate.cc
index d82925a..27d4161 100644
--- a/audio/channel_receive_frame_transformer_delegate.cc
+++ b/audio/channel_receive_frame_transformer_delegate.cc
@@ -22,9 +22,11 @@
#include "api/scoped_refptr.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
+#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "rtc_base/buffer.h"
#include "rtc_base/string_encode.h"
+#include "system_wrappers/include/ntp_time.h"
namespace webrtc {
@@ -96,6 +98,25 @@
: std::optional<Timestamp>(receive_time_);
}
+ std::optional<Timestamp> CaptureTime() const override {
+ if (header_.extension.absolute_capture_time) {
+ return Timestamp::Micros(UQ32x32ToInt64Us(
+ header_.extension.absolute_capture_time->absolute_capture_timestamp));
+ }
+ return std::nullopt;
+ }
+
+ std::optional<TimeDelta> SenderCaptureTimeOffset() const override {
+ if (header_.extension.absolute_capture_time &&
+ header_.extension.absolute_capture_time
+ ->estimated_capture_clock_offset) {
+ return TimeDelta::Micros(
+ UQ32x32ToInt64Us(*header_.extension.absolute_capture_time
+ ->estimated_capture_clock_offset));
+ }
+ return std::nullopt;
+ }
+
private:
rtc::Buffer payload_;
RTPHeader header_;
diff --git a/audio/channel_receive_frame_transformer_delegate_unittest.cc b/audio/channel_receive_frame_transformer_delegate_unittest.cc
index 32e56c4..dad1f63 100644
--- a/audio/channel_receive_frame_transformer_delegate_unittest.cc
+++ b/audio/channel_receive_frame_transformer_delegate_unittest.cc
@@ -24,6 +24,7 @@
#include "api/test/mock_transformable_audio_frame.h"
#include "api/units/timestamp.h"
#include "rtc_base/thread.h"
+#include "system_wrappers/include/ntp_time.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -193,7 +194,7 @@
}
TEST(ChannelReceiveFrameTransformerDelegateTest,
- AudioLevelAbsentWithoutExtension) {
+ AudioLevelAndCaptureTimeAbsentWithoutExtension) {
rtc::AutoThread main_thread;
rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
@@ -223,6 +224,8 @@
auto* audio_frame =
static_cast<TransformableAudioFrameInterface*>(frame.get());
EXPECT_FALSE(audio_frame->AudioLevel());
+ EXPECT_FALSE(audio_frame->CaptureTime());
+ EXPECT_FALSE(audio_frame->SenderCaptureTimeOffset());
EXPECT_EQ(audio_frame->Type(),
TransformableAudioFrameInterface::FrameType::kAudioFrameCN);
}
@@ -265,5 +268,48 @@
TransformableAudioFrameInterface::FrameType::kAudioFrameSpeech);
}
+TEST(ChannelReceiveFrameTransformerDelegateTest,
+ CaptureTimePresentWithExtension) {
+ rtc::AutoThread main_thread;
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
+ rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate =
+ rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>(
+ /*receive_frame_callback=*/nullptr, mock_frame_transformer,
+ rtc::Thread::Current());
+ rtc::scoped_refptr<TransformedFrameCallback> callback;
+ EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback)
+ .WillOnce(SaveArg<0>(&callback));
+ delegate->Init();
+ ASSERT_TRUE(callback);
+
+ const uint8_t data[] = {1, 2, 3, 4};
+ rtc::ArrayView<const uint8_t> packet(data, sizeof(data));
+ Timestamp capture_time = Timestamp::Millis(1234);
+ TimeDelta sender_capture_time_offset = TimeDelta::Millis(56);
+ AbsoluteCaptureTime absolute_capture_time = {
+ .absolute_capture_timestamp = Int64MsToUQ32x32(capture_time.ms()),
+ .estimated_capture_clock_offset =
+ Int64MsToUQ32x32(sender_capture_time_offset.ms())};
+ RTPHeader header;
+ header.extension.absolute_capture_time = absolute_capture_time;
+
+ std::unique_ptr<TransformableFrameInterface> frame;
+ ON_CALL(*mock_frame_transformer, Transform)
+ .WillByDefault(
+ [&](std::unique_ptr<TransformableFrameInterface> transform_frame) {
+ frame = std::move(transform_frame);
+ });
+ delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
+ kFakeReceiveTimestamp);
+
+ EXPECT_TRUE(frame);
+ auto* audio_frame =
+ static_cast<TransformableAudioFrameInterface*>(frame.get());
+ EXPECT_EQ(*audio_frame->CaptureTime(), capture_time);
+ EXPECT_EQ(*audio_frame->SenderCaptureTimeOffset(),
+ sender_capture_time_offset);
+}
+
} // namespace
} // namespace webrtc
diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc
index dacbfcd..53d8648 100644
--- a/audio/channel_send_frame_transformer_delegate.cc
+++ b/audio/channel_send_frame_transformer_delegate.cc
@@ -13,6 +13,9 @@
#include <utility>
#include <vector>
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+
namespace webrtc {
namespace {
@@ -110,6 +113,10 @@
}
std::optional<Timestamp> ReceiveTime() const override { return std::nullopt; }
+ std::optional<Timestamp> CaptureTime() const override { return std::nullopt; }
+ std::optional<TimeDelta> SenderCaptureTimeOffset() const override {
+ return std::nullopt;
+ }
private:
AudioFrameType frame_type_;
