Add CreateOutgoingAudioFrame. A new API for creating EncodedAudioFrames that are not linked to any particular sender. Add support in the audio sender to handle such frames. Migrate all calls from TransformableFrameInterface::GetTimestamp to TransformableFrameInterface::GetRtpTimestampInfo. Bug: chromium:524901718 Change-Id: I64f29ea2cdf442f160b2c6c76fd50cb4d5912a82 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/485761 Reviewed-by: Jakob Ivarsson‎ <jakobi@webrtc.org> Reviewed-by: Harald Alvestrand <hta@webrtc.org> Commit-Queue: Leonardo Evi <evil@chromium.org> Cr-Commit-Position: refs/heads/main@{#48109}
diff --git a/api/BUILD.gn b/api/BUILD.gn index ac60d88..44661dc 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn
@@ -1757,6 +1757,7 @@ "rtp_parameters_unittest.cc", "scoped_refptr_unittest.cc", "sequence_checker_unittest.cc", + "test/frame_transformer_factory_unittest.cc", "test/peerconnection_quality_test_fixture_unittest.cc", "webrtc_sdp_encoding_options_unittest.cc", "webrtc_sdp_unittest.cc", @@ -1766,8 +1767,13 @@ ":candidate", ":field_trials", ":field_trials_view", + ":frame_transformer_factory", + ":frame_transformer_interface", ":function_view", ":jsep", + ":mock_transformable_audio_frame", + ":mock_transformable_video_frame", + ":payload_type", ":peer_connection_interface", ":rtc_error", ":rtc_event_log_output_file", @@ -1824,6 +1830,7 @@ "video:video_broadcaster_unittest", "video:video_common_unittest", "video:video_frame", + "video:video_frame_metadata", "video:video_frame_metadata_unittest", "video/corruption_detection:corruption_detection_unittests", "video/corruption_detection:frame_instrumentation_data_reader_unittest", @@ -1941,6 +1948,7 @@ ] deps = [ ":frame_transformer_interface", + ":payload_type", "../audio", "../modules/rtp_rtcp", "../rtc_base:checks",
diff --git a/api/frame_transformer_factory.cc b/api/frame_transformer_factory.cc index ed2c566..3b0cc00 100644 --- a/api/frame_transformer_factory.cc +++ b/api/frame_transformer_factory.cc
@@ -10,9 +10,15 @@ #include "api/frame_transformer_factory.h" +#include <cstddef> +#include <cstdint> #include <memory> +#include <optional> +#include <string> +#include <vector> #include "api/frame_transformer_interface.h" +#include "api/payload_type.h" #include "audio/channel_receive_frame_transformer_delegate.h" #include "audio/channel_send_frame_transformer_delegate.h" #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" @@ -34,4 +40,23 @@ return CloneSenderVideoFrame(original); } +std::unique_ptr<TransformableAudioFrameInterface> CreateOutgoingAudioFrame( + TransformableAudioFrameInterface::FrameType frame_type, + PayloadType payload_type, + uint32_t rtp_timestamp_without_offset, + const uint8_t* payload_data, + size_t payload_size, + std::optional<uint64_t> absolute_capture_timestamp_ms, + uint32_t ssrc, + const std::vector<uint32_t>& csrcs, + const std::string& codec_mime_type, + std::optional<uint16_t> sequence_number, + std::optional<uint8_t> audio_level_dbov) { + return CreateSenderAudioFrame( + frame_type, payload_type.value(), + RtpTimestampWithoutOffset{rtp_timestamp_without_offset}, payload_data, + payload_size, absolute_capture_timestamp_ms, ssrc, csrcs, codec_mime_type, + sequence_number, audio_level_dbov); +} + } // namespace webrtc
diff --git a/api/frame_transformer_factory.h b/api/frame_transformer_factory.h index 602a740..27595f9 100644 --- a/api/frame_transformer_factory.h +++ b/api/frame_transformer_factory.h
@@ -11,9 +11,15 @@ #ifndef API_FRAME_TRANSFORMER_FACTORY_H_ #define API_FRAME_TRANSFORMER_FACTORY_H_ +#include <cstddef> +#include <cstdint> #include <memory> +#include <optional> +#include <string> +#include <vector> #include "api/frame_transformer_interface.h" +#include "api/payload_type.h" #include "rtc_base/system/rtc_export.h" // This file contains EXPERIMENTAL functions to create video frames from @@ -28,6 +34,18 @@ TransformableAudioFrameInterface* original); RTC_EXPORT std::unique_ptr<TransformableVideoFrameInterface> CloneVideoFrame( TransformableVideoFrameInterface* original); +RTC_EXPORT std::unique_ptr<TransformableAudioFrameInterface> +CreateOutgoingAudioFrame(TransformableAudioFrameInterface::FrameType frame_type, + PayloadType payload_type, + uint32_t rtp_timestamp_without_offset, + const uint8_t* payload_data, + size_t payload_size, + std::optional<uint64_t> absolute_capture_timestamp_ms, + uint32_t ssrc, + const std::vector<uint32_t>& csrcs, + const std::string& codec_mime_type, + std::optional<uint16_t> sequence_number, + std::optional<uint8_t> audio_level_dbov); } // namespace webrtc #endif // API_FRAME_TRANSFORMER_FACTORY_H_
diff --git a/api/test/frame_transformer_factory_unittest.cc b/api/test/frame_transformer_factory_unittest.cc new file mode 100644 index 0000000..2251fdc --- /dev/null +++ b/api/test/frame_transformer_factory_unittest.cc
@@ -0,0 +1,95 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/frame_transformer_factory.h" + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <optional> +#include <span> +#include <variant> +#include <vector> + +#include "api/frame_transformer_interface.h" +#include "api/payload_type.h" +#include "api/test/mock_transformable_audio_frame.h" +#include "api/test/mock_transformable_video_frame.h" +#include "api/video/video_frame_metadata.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using testing::Each; +using testing::ElementsAreArray; +using testing::NiceMock; +using testing::Return; + +TEST(FrameTransformerFactory, CloneAudioFrame) { + NiceMock<MockTransformableAudioFrame> original_frame; + uint8_t data[10]; + std::fill_n(data, 10, 5); + std::span<uint8_t> data_view(data); + ON_CALL(original_frame, GetData()).WillByDefault(Return(data_view)); + auto cloned_frame = CloneAudioFrame(&original_frame); + + EXPECT_THAT(cloned_frame->GetData(), ElementsAreArray(data)); +} + +TEST(FrameTransformerFactory, CloneVideoFrame) { + NiceMock<MockTransformableVideoFrame> original_frame; + uint8_t data[10]; + std::fill_n(data, 10, 5); + std::span<uint8_t> data_view(data); + EXPECT_CALL(original_frame, GetData()).WillRepeatedly(Return(data_view)); + VideoFrameMetadata metadata; + std::vector<uint32_t> csrcs{123, 321}; + // Copy csrcs rather than moving so we can compare in an EXPECT_EQ later. + metadata.SetCsrcs(csrcs); + + EXPECT_CALL(original_frame, Metadata()).WillRepeatedly(Return(metadata)); + auto cloned_frame = CloneVideoFrame(&original_frame); + + EXPECT_EQ(cloned_frame->GetData().size(), 10u); + EXPECT_THAT(cloned_frame->GetData(), Each(5u)); + EXPECT_EQ(cloned_frame->Metadata().GetCsrcs(), csrcs); +} + +TEST(FrameTransformerFactory, CreateOutgoingAudioFrame) { + uint8_t data[] = {1, 2, 3, 4}; + std::vector<uint32_t> csrcs{123, 321}; + auto frame = CreateOutgoingAudioFrame( + TransformableAudioFrameInterface::FrameType::kAudioFrameSpeech, + PayloadType(111), /*rtp_timestamp_without_offset=*/222, data, + sizeof(data), /*absolute_capture_timestamp_ms=*/333, /*ssrc=*/444, csrcs, + "audio/opus", /*sequence_number=*/555, /*audio_level_dbov=*/67); + + ASSERT_TRUE(frame); + EXPECT_EQ(frame->Type(), + TransformableAudioFrameInterface::FrameType::kAudioFrameSpeech); + EXPECT_EQ(frame->GetPayloadType(), 111u); + EXPECT_THAT(frame->GetData(), ElementsAreArray(data)); + EXPECT_EQ(frame->AbsoluteCaptureTimestamp(), + std::make_optional<uint64_t>(333)); + EXPECT_EQ(frame->GetSsrc(), 444u); + EXPECT_THAT(frame->GetContributingSources(), ElementsAreArray(csrcs)); + EXPECT_EQ(frame->GetMimeType(), "audio/opus"); + EXPECT_EQ(frame->SequenceNumber(), std::make_optional<uint16_t>(555)); + EXPECT_EQ(frame->AudioLevel(), std::make_optional<uint8_t>(67)); + EXPECT_TRUE(std::holds_alternative<RtpTimestampWithoutOffset>( + frame->GetRtpTimestampInfo())); + EXPECT_EQ(std::get<RtpTimestampWithoutOffset>(frame->GetRtpTimestampInfo()), + 222u); +} + +} // namespace +} // namespace webrtc
diff --git a/audio/channel_receive_frame_transformer_delegate_unittest.cc b/audio/channel_receive_frame_transformer_delegate_unittest.cc index 78bfeff..67a46a0 100644 --- a/audio/channel_receive_frame_transformer_delegate_unittest.cc +++ b/audio/channel_receive_frame_transformer_delegate_unittest.cc
@@ -388,7 +388,6 @@ // Test the SetRTPTimestamp setter uint32_t new_timestamp = 112233u; audio_frame->SetRTPTimestamp(new_timestamp); - EXPECT_EQ(audio_frame->GetTimestamp(), new_timestamp); EXPECT_TRUE(std::holds_alternative<RtpTimestampWithOffset>( audio_frame->GetRtpTimestampInfo())); EXPECT_EQ(
diff --git a/audio/channel_send.cc b/audio/channel_send.cc index 591b769..324a188 100644 --- a/audio/channel_send.cc +++ b/audio/channel_send.cc
@@ -19,6 +19,7 @@ #include <span> #include <string> #include <utility> +#include <variant> #include <vector> #include "absl/functional/any_invocable.h" @@ -982,16 +983,25 @@ // to send the transformed audio. ChannelSendFrameTransformerDelegate::SendFrameCallback send_audio_callback = [this](AudioFrameType frameType, uint8_t payloadType, - uint32_t rtp_timestamp_with_offset, + RtpTimestampInfo rtp_timestamp_info, std::span<const uint8_t> payload, int64_t absolute_capture_timestamp_ms, std::span<const uint32_t> csrcs, std::optional<uint8_t> audio_level_dbov) { RTC_DCHECK_RUN_ON(worker_thread_); - return SendRtpAudio( - frameType, payloadType, - rtp_timestamp_with_offset - rtp_rtcp_->StartTimestamp(), payload, - absolute_capture_timestamp_ms, csrcs, audio_level_dbov); + uint32_t timestamp_without_offset; + if (std::holds_alternative<RtpTimestampWithoutOffset>( + rtp_timestamp_info)) { + timestamp_without_offset = + std::get<RtpTimestampWithoutOffset>(rtp_timestamp_info); + } else { + timestamp_without_offset = + std::get<RtpTimestampWithOffset>(rtp_timestamp_info) - + rtp_rtcp_->StartTimestamp(); + } + return SendRtpAudio(frameType, payloadType, timestamp_without_offset, + payload, absolute_capture_timestamp_ms, csrcs, + audio_level_dbov); }; frame_transformer_delegate_ = make_ref_counted<ChannelSendFrameTransformerDelegate>(
diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc index dfe0f9f..670d996 100644 --- a/audio/channel_send_frame_transformer_delegate.cc +++ b/audio/channel_send_frame_transformer_delegate.cc
@@ -17,6 +17,7 @@ #include <span> #include <string> #include <utility> +#include <variant> #include <vector> #include "api/frame_transformer_interface.h" @@ -79,10 +80,35 @@ const std::string& codec_mime_type, std::optional<uint16_t> sequence_number, std::optional<uint8_t> audio_level_dbov) + : TransformableOutgoingAudioFrame( + frame_type, + payload_type, + RtpTimestampInfo(RtpTimestampWithOffset{rtp_timestamp_with_offset}), + payload_data, + payload_size, + absolute_capture_timestamp_ms, + ssrc, + std::move(csrcs), + codec_mime_type, + sequence_number, + audio_level_dbov) {} + + TransformableOutgoingAudioFrame( + AudioFrameType frame_type, + uint8_t payload_type, + RtpTimestampInfo rtp_timestamp_info, + const uint8_t* payload_data, + size_t payload_size, + std::optional<uint64_t> absolute_capture_timestamp_ms, + uint32_t ssrc, + std::vector<uint32_t> csrcs, + const std::string& codec_mime_type, + std::optional<uint16_t> sequence_number, + std::optional<uint8_t> audio_level_dbov) : TransformableAudioFrameInterface(Passkey()), frame_type_(frame_type), payload_type_(payload_type), - rtp_timestamp_with_offset_(rtp_timestamp_with_offset), + rtp_timestamp_info_(rtp_timestamp_info), payload_(payload_data, payload_size), absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms), ssrc_(ssrc), @@ -90,12 +116,18 @@ codec_mime_type_(codec_mime_type), sequence_number_(sequence_number), audio_level_dbov_(audio_level_dbov) {} + ~TransformableOutgoingAudioFrame() override = default; std::span<const uint8_t> GetData() const override { return payload_; } void SetData(std::span<const uint8_t> data) override { payload_.SetData(data.data(), data.size()); } - uint32_t GetTimestamp() const override { return rtp_timestamp_with_offset_; } + uint32_t GetTimestamp() const override { + if (std::holds_alternative<RtpTimestampWithOffset>(rtp_timestamp_info_)) { + return std::get<RtpTimestampWithOffset>(rtp_timestamp_info_); + } + return 0; + } uint32_t GetSsrc() const override { return ssrc_; } IfaceFrameType Type() const override { @@ -119,7 +151,7 @@ } void SetRTPTimestamp(uint32_t rtp_timestamp_with_offset) override { - rtp_timestamp_with_offset_ = rtp_timestamp_with_offset; + rtp_timestamp_info_ = RtpTimestampWithOffset{rtp_timestamp_with_offset}; } std::optional<uint64_t> AbsoluteCaptureTimestamp() const override { @@ -156,13 +188,13 @@ } RtpTimestampInfo GetRtpTimestampInfo() const override { - return RtpTimestampWithOffset{rtp_timestamp_with_offset_}; + return rtp_timestamp_info_; } private: AudioFrameType frame_type_; uint8_t payload_type_; - uint32_t rtp_timestamp_with_offset_; + RtpTimestampInfo rtp_timestamp_info_; Buffer payload_; std::optional<uint64_t> absolute_capture_timestamp_ms_; uint32_t ssrc_; @@ -249,8 +281,8 @@ static_cast<TransformableAudioFrameInterface*>(frame.get()); send_frame_callback_( InterfaceFrameTypeToInternalFrameType(transformed_frame->Type()), - transformed_frame->GetPayloadType(), transformed_frame->GetTimestamp(), - transformed_frame->GetData(), + transformed_frame->GetPayloadType(), + transformed_frame->GetRtpTimestampInfo(), transformed_frame->GetData(), transformed_frame->AbsoluteCaptureTimestamp() ? *transformed_frame->AbsoluteCaptureTimestamp() : 0, @@ -265,11 +297,30 @@ original->GetContributingSources().end()); return std::make_unique<TransformableOutgoingAudioFrame>( InterfaceFrameTypeToInternalFrameType(original->Type()), - original->GetPayloadType(), original->GetTimestamp(), + original->GetPayloadType(), original->GetRtpTimestampInfo(), original->GetData().data(), original->GetData().size(), original->AbsoluteCaptureTimestamp(), original->GetSsrc(), std::move(csrcs), original->GetMimeType(), original->SequenceNumber(), original->AudioLevel()); } +std::unique_ptr<TransformableAudioFrameInterface> CreateSenderAudioFrame( + TransformableAudioFrameInterface::FrameType frame_type, + uint8_t payload_type, + RtpTimestampInfo rtp_timestamp_info, + const uint8_t* payload_data, + size_t payload_size, + std::optional<uint64_t> absolute_capture_timestamp_ms, + uint32_t ssrc, + const std::vector<uint32_t>& csrcs, + const std::string& codec_mime_type, + std::optional<uint16_t> sequence_number, + std::optional<uint8_t> audio_level_dbov) { + return std::make_unique<TransformableOutgoingAudioFrame>( + InterfaceFrameTypeToInternalFrameType(frame_type), payload_type, + rtp_timestamp_info, payload_data, payload_size, + absolute_capture_timestamp_ms, ssrc, csrcs, codec_mime_type, + sequence_number, audio_level_dbov); +} + } // namespace webrtc
diff --git a/audio/channel_send_frame_transformer_delegate.h b/audio/channel_send_frame_transformer_delegate.h index 678f51f..ba28554 100644 --- a/audio/channel_send_frame_transformer_delegate.h +++ b/audio/channel_send_frame_transformer_delegate.h
@@ -39,7 +39,7 @@ using SendFrameCallback = std::function<int32_t(AudioFrameType frameType, uint8_t payloadType, - uint32_t rtp_timestamp_with_offset, + RtpTimestampInfo rtp_timestamp_info, std::span<const uint8_t> payload, int64_t absolute_capture_timestamp_ms, std::span<const uint32_t> csrcs, @@ -94,6 +94,17 @@ std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame( TransformableAudioFrameInterface* original); - +std::unique_ptr<TransformableAudioFrameInterface> CreateSenderAudioFrame( + TransformableAudioFrameInterface::FrameType frame_type, + uint8_t payload_type, + RtpTimestampInfo rtp_timestamp_info, + const uint8_t* payload_data, + size_t payload_size, + std::optional<uint64_t> absolute_capture_timestamp_ms, + uint32_t ssrc, + const std::vector<uint32_t>& csrcs, + const std::string& codec_mime_type, + std::optional<uint16_t> sequence_number, + std::optional<uint8_t> audio_level_dbov); } // namespace webrtc #endif // AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_
diff --git a/audio/channel_send_frame_transformer_delegate_unittest.cc b/audio/channel_send_frame_transformer_delegate_unittest.cc index 58c2933..927f7c6 100644 --- a/audio/channel_send_frame_transformer_delegate_unittest.cc +++ b/audio/channel_send_frame_transformer_delegate_unittest.cc
@@ -51,7 +51,7 @@ SendFrame, (AudioFrameType frameType, uint8_t payloadType, - uint32_t rtp_timestamp, + RtpTimestampInfo rtp_timestamp_info, std::span<const uint8_t> payload, int64_t absolute_capture_timestamp_ms, std::span<const uint32_t> csrcs, @@ -59,11 +59,12 @@ ChannelSendFrameTransformerDelegate::SendFrameCallback callback() { return [this](AudioFrameType frameType, uint8_t payloadType, - uint32_t rtp_timestamp, std::span<const uint8_t> payload, + RtpTimestampInfo rtp_timestamp_info, + std::span<const uint8_t> payload, int64_t absolute_capture_timestamp_ms, std::span<const uint32_t> csrcs, std::optional<uint8_t> audio_level_dbov) { - return SendFrame(frameType, payloadType, rtp_timestamp, payload, + return SendFrame(frameType, payloadType, rtp_timestamp_info, payload, absolute_capture_timestamp_ms, csrcs, audio_level_dbov); }; } @@ -186,8 +187,9 @@ const uint8_t audio_level_dbov = 17; EXPECT_CALL(mock_channel, SendFrame).Times(0); EXPECT_CALL(mock_channel, - SendFrame(_, 0, 0, ElementsAreArray(mock_data), _, - ElementsAreArray(csrcs), Optional(audio_level_dbov))); + SendFrame(_, 0, RtpTimestampInfo(RtpTimestampWithOffset{0u}), + ElementsAreArray(mock_data), _, ElementsAreArray(csrcs), + Optional(audio_level_dbov))); ON_CALL(*mock_frame_transformer, Transform) .WillByDefault( [&](std::unique_ptr<TransformableFrameInterface> /* frame */) { @@ -301,7 +303,7 @@ std::unique_ptr<TransformableAudioFrameInterface> cloned_frame = CloneSenderAudioFrame(frame.get()); - EXPECT_EQ(cloned_frame->GetTimestamp(), frame->GetTimestamp()); + EXPECT_EQ(cloned_frame->GetRtpTimestampInfo(), frame->GetRtpTimestampInfo()); EXPECT_EQ(cloned_frame->GetSsrc(), frame->GetSsrc()); EXPECT_EQ(cloned_frame->Type(), frame->Type()); EXPECT_EQ(cloned_frame->GetPayloadType(), frame->GetPayloadType()); @@ -318,7 +320,7 @@ std::unique_ptr<TransformableAudioFrameInterface> cloned_frame = CloneSenderAudioFrame(frame.get()); - EXPECT_EQ(cloned_frame->GetTimestamp(), frame->GetTimestamp()); + EXPECT_EQ(cloned_frame->GetRtpTimestampInfo(), frame->GetRtpTimestampInfo()); EXPECT_EQ(cloned_frame->GetSsrc(), frame->GetSsrc()); EXPECT_EQ(cloned_frame->Type(), frame->Type()); EXPECT_EQ(cloned_frame->GetPayloadType(), frame->GetPayloadType()); @@ -374,7 +376,6 @@ audio_frame->SetRTPTimestamp(new_timestamp); // Test the getter after - EXPECT_EQ(audio_frame->GetTimestamp(), new_timestamp); EXPECT_TRUE(std::holds_alternative<RtpTimestampWithOffset>( audio_frame->GetRtpTimestampInfo())); EXPECT_EQ( @@ -382,5 +383,105 @@ new_timestamp); } +TEST(ChannelSendFrameTransformerDelegateTest, + TransformPropagatesRtpTimestampWithoutOffset) { + TaskQueueForTest channel_queue("channel_queue"); + scoped_refptr<MockFrameTransformer> mock_frame_transformer = + make_ref_counted<NiceMock<MockFrameTransformer>>(); + MockChannelSend mock_channel; + scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = + make_ref_counted<ChannelSendFrameTransformerDelegate>( + mock_channel.callback(), mock_frame_transformer, channel_queue.Get()); + + EXPECT_CALL( + mock_channel, + SendFrame(_, _, RtpTimestampInfo(RtpTimestampWithoutOffset{1234u}), _, _, + _, _)); + + std::unique_ptr<TransformableAudioFrameInterface> frame = + CreateSenderAudioFrame( + TransformableAudioFrameInterface::FrameType::kEmptyFrame, 0, + RtpTimestampInfo(RtpTimestampWithoutOffset{1234u}), mock_data, + sizeof(mock_data), + /*absolute_capture_timestamp_ms=*/0, /*ssrc=*/0, /*csrcs=*/{}, + /*codec_mime_type=*/"audio/opus", /*sequence_number=*/std::nullopt, + /*audio_level_dbov=*/std::nullopt); + + delegate->OnTransformedFrame(std::move(frame)); + channel_queue.WaitForPreviouslyPostedTasks(); +} + +TEST(ChannelSendFrameTransformerDelegateTest, + TransformPropagatesRtpTimestampWithOffset) { + TaskQueueForTest channel_queue("channel_queue"); + scoped_refptr<MockFrameTransformer> mock_frame_transformer = + make_ref_counted<NiceMock<MockFrameTransformer>>(); + MockChannelSend mock_channel; + scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = + make_ref_counted<ChannelSendFrameTransformerDelegate>( + mock_channel.callback(), mock_frame_transformer, channel_queue.Get()); + + EXPECT_CALL(mock_channel, + SendFrame(_, _, RtpTimestampInfo(RtpTimestampWithOffset{4321u}), + _, _, _, _)); + + std::unique_ptr<TransformableAudioFrameInterface> frame = + CreateSenderAudioFrame( + TransformableAudioFrameInterface::FrameType::kEmptyFrame, 0, + RtpTimestampInfo(RtpTimestampWithOffset{4321u}), mock_data, + sizeof(mock_data), + /*absolute_capture_timestamp_ms=*/0, /*ssrc=*/0, /*csrcs=*/{}, + /*codec_mime_type=*/"audio/opus", /*sequence_number=*/std::nullopt, + /*audio_level_dbov=*/std::nullopt); + + delegate->OnTransformedFrame(std::move(frame)); + channel_queue.WaitForPreviouslyPostedTasks(); +} + +TEST(ChannelSendFrameTransformerDelegateTest, + CreateSenderAudioFrameFieldsMatch) { + uint8_t data[] = {10, 20, 30, 40}; + std::vector<uint32_t> csrcs{12, 34}; + std::unique_ptr<TransformableAudioFrameInterface> frame = + CreateSenderAudioFrame( + TransformableAudioFrameInterface::FrameType::kAudioFrameSpeech, 12, + RtpTimestampInfo(RtpTimestampWithoutOffset{12345u}), data, + sizeof(data), + /*absolute_capture_timestamp_ms=*/67890, /*ssrc=*/13579, csrcs, + /*codec_mime_type=*/"mime/type", /*sequence_number=*/9876, + /*audio_level_dbov=*/45); + + ASSERT_TRUE(frame); + EXPECT_EQ(frame->Type(), + TransformableAudioFrameInterface::FrameType::kAudioFrameSpeech); + EXPECT_EQ(frame->GetPayloadType(), 12u); + EXPECT_THAT(frame->GetData(), ElementsAreArray(data)); + EXPECT_EQ(frame->AbsoluteCaptureTimestamp(), + std::make_optional<uint64_t>(67890)); + EXPECT_EQ(frame->GetSsrc(), 13579u); + EXPECT_THAT(frame->GetContributingSources(), ElementsAreArray(csrcs)); + EXPECT_EQ(frame->GetMimeType(), "mime/type"); + EXPECT_EQ(frame->SequenceNumber(), std::make_optional<uint16_t>(9876)); + EXPECT_EQ(frame->AudioLevel(), std::make_optional<uint8_t>(45)); + EXPECT_TRUE(std::holds_alternative<RtpTimestampWithoutOffset>( + frame->GetRtpTimestampInfo())); + EXPECT_EQ(std::get<RtpTimestampWithoutOffset>(frame->GetRtpTimestampInfo()), + 12345u); + + std::unique_ptr<TransformableAudioFrameInterface> frame_with_offset = + CreateSenderAudioFrame( + TransformableAudioFrameInterface::FrameType::kEmptyFrame, 0, + RtpTimestampInfo(RtpTimestampWithOffset{54321u}), mock_data, + sizeof(mock_data), + /*absolute_capture_timestamp_ms=*/0, /*ssrc=*/0, /*csrcs=*/{}, + /*codec_mime_type=*/"audio/opus", /*sequence_number=*/std::nullopt, + /*audio_level_dbov=*/std::nullopt); + EXPECT_TRUE(std::holds_alternative<RtpTimestampWithOffset>( + frame_with_offset->GetRtpTimestampInfo())); + EXPECT_EQ(std::get<RtpTimestampWithOffset>( + frame_with_offset->GetRtpTimestampInfo()), + 54321u); +} + } // namespace } // namespace webrtc
diff --git a/audio/channel_send_unittest.cc b/audio/channel_send_unittest.cc index 69eaad2..abd6a52 100644 --- a/audio/channel_send_unittest.cc +++ b/audio/channel_send_unittest.cc
@@ -16,6 +16,7 @@ #include <optional> #include <span> #include <utility> +#include <variant> #include <vector> #include "api/audio/audio_frame.h" @@ -208,7 +209,10 @@ int64_t transformable_frame_timestamp = -1; EXPECT_CALL(*mock_frame_transformer, Transform) .WillOnce([&](std::unique_ptr<TransformableFrameInterface> frame) { - transformable_frame_timestamp = frame->GetTimestamp(); + ASSERT_TRUE(std::holds_alternative<RtpTimestampWithOffset>( + frame->GetRtpTimestampInfo())); + transformable_frame_timestamp = + std::get<RtpTimestampWithOffset>(frame->GetRtpTimestampInfo()); callback->OnTransformedFrame(std::move(frame)); }); // Insert two frames which should trigger a new packet. @@ -449,8 +453,8 @@ frame_template->GetData().begin(), frame_template->GetData().end()); ON_CALL(*mock_frame, GetData).WillByDefault(Return(frame_data)); - ON_CALL(*mock_frame, GetTimestamp) - .WillByDefault(Return(frame_template->GetTimestamp())); + ON_CALL(*mock_frame, GetRtpTimestampInfo) + .WillByDefault(Return(frame_template->GetRtpTimestampInfo())); ON_CALL(*mock_frame, GetPayloadType) .WillByDefault(Return(frame_template->GetPayloadType())); ON_CALL(*mock_frame, GetSsrc) @@ -525,6 +529,94 @@ "packet."; } +TEST_F(ChannelSendTest, CallbackOnFrameWithRtpOffset) { + scoped_refptr<MockFrameTransformer> mock_frame_transformer = + make_ref_counted<MockFrameTransformer>(); + channel_->SetEncoderToPacketizerFrameTransformer(mock_frame_transformer); + + scoped_refptr<TransformedFrameCallback> callback; + EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback) + .WillOnce(SaveArg<0>(&callback)); + EXPECT_CALL(*mock_frame_transformer, UnregisterTransformedFrameCallback); + + std::optional<uint32_t> sent_timestamp; + auto send_rtp = [&](std::span<const uint8_t> data, + const PacketOptions& /* options */) { + RtpPacketReceived packet; + packet.Parse(data); + sent_timestamp = packet.Timestamp(); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(send_rtp); + + channel_->StartSend(); + // Wait for callback registration task to run. + EXPECT_THAT(WaitUntil([&] { return callback != nullptr; }, IsTrue(), + {.clock = &time_controller_}), + IsRtcOk()); + + // Call with RtpTimestampWithOffset + auto mock_frame = std::make_unique<NiceMock<MockTransformableAudioFrame>>(); + ON_CALL(*mock_frame, GetRtpTimestampInfo) + .WillByDefault(Return(RtpTimestampInfo{RtpTimestampWithOffset{54321u}})); + uint8_t payload[10] = {0}; + ON_CALL(*mock_frame, GetData) + .WillByDefault(Return(std::span<const uint8_t>(&payload[0], 10))); + callback->OnTransformedFrame(std::move(mock_frame)); + + EXPECT_THAT(WaitUntil([&] { return sent_timestamp; }, IsTrue(), + {.clock = &time_controller_}), + IsRtcOk()); + // Frame already contains the offset. Expected on-the-wire value is the same. + EXPECT_EQ(*sent_timestamp, 54321u); +} + +TEST_F(ChannelSendTest, CallbackOnFrameWithoutRtpOffset) { + scoped_refptr<MockFrameTransformer> mock_frame_transformer = + make_ref_counted<MockFrameTransformer>(); + channel_->SetEncoderToPacketizerFrameTransformer(mock_frame_transformer); + + scoped_refptr<TransformedFrameCallback> callback; + EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback) + .WillOnce(SaveArg<0>(&callback)); + EXPECT_CALL(*mock_frame_transformer, UnregisterTransformedFrameCallback); + + std::optional<uint32_t> sent_timestamp; + auto send_rtp = [&](std::span<const uint8_t> data, + const PacketOptions& /* options */) { + RtpPacketReceived packet; + packet.Parse(data); + sent_timestamp = packet.Timestamp(); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(send_rtp); + + channel_->StartSend(); + // Wait for callback registration task to run. + EXPECT_THAT(WaitUntil([&] { return callback != nullptr; }, IsTrue(), + {.clock = &time_controller_}), + IsRtcOk()); + + uint32_t start_timestamp = channel_->GetRtpRtcp()->StartTimestamp(); + + // Call with RtpTimestampWithoutOffset + auto mock_frame = std::make_unique<NiceMock<MockTransformableAudioFrame>>(); + ON_CALL(*mock_frame, GetRtpTimestampInfo) + .WillByDefault( + Return(RtpTimestampInfo{RtpTimestampWithoutOffset{1234u}})); + uint8_t payload[10] = {0}; + ON_CALL(*mock_frame, GetData) + .WillByDefault(Return(std::span<const uint8_t>(&payload[0], 10))); + callback->OnTransformedFrame(std::move(mock_frame)); + + EXPECT_THAT(WaitUntil([&] { return sent_timestamp; }, IsTrue(), + {.clock = &time_controller_}), + IsRtcOk()); + // Frame did not contain the offset. Expected on-the-wire value must include + // the offset. + EXPECT_EQ(*sent_timestamp, 1234u + start_timestamp); +} + } // namespace } // namespace voe } // namespace webrtc
diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 5541046..c73c903 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn
@@ -641,7 +641,6 @@ deps = [ ":corruption_detection_extension_unittest", ":fec_test_helper", - ":frame_transformer_factory_unittest", ":leb128", ":mock_rtp_rtcp", ":ntp_time_util", @@ -730,19 +729,6 @@ ] } - rtc_library("frame_transformer_factory_unittest") { - testonly = true - sources = [ "source/frame_transformer_factory_unittest.cc" ] - deps = [ - "../../api:frame_transformer_factory", - "../../api:mock_transformable_audio_frame", - "../../api:mock_transformable_video_frame", - "../../api/video:video_frame_metadata", - "../../test:test_support", - "//third_party/abseil-cpp/absl/memory", - ] - } - rtc_library("corruption_detection_extension_unittest") { testonly = true sources = [ "source/corruption_detection_extension_unittest.cc" ]
diff --git a/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc b/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc deleted file mode 100644 index 0e909c7..0000000 --- a/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc +++ /dev/null
@@ -1,64 +0,0 @@ -/* - * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "api/frame_transformer_factory.h" - -#include <algorithm> -#include <cstdint> -#include <memory> -#include <span> -#include <vector> - -#include "api/test/mock_transformable_audio_frame.h" -#include "api/test/mock_transformable_video_frame.h" -#include "api/video/video_frame_metadata.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace { - -using testing::Each; -using testing::ElementsAreArray; -using testing::NiceMock; -using testing::Return; - -TEST(FrameTransformerFactory, CloneAudioFrame) { - NiceMock<MockTransformableAudioFrame> original_frame; - uint8_t data[10]; - std::fill_n(data, 10, 5); - std::span<uint8_t> data_view(data); - ON_CALL(original_frame, GetData()).WillByDefault(Return(data_view)); - auto cloned_frame = CloneAudioFrame(&original_frame); - - EXPECT_THAT(cloned_frame->GetData(), ElementsAreArray(data)); -} - -TEST(FrameTransformerFactory, CloneVideoFrame) { - NiceMock<MockTransformableVideoFrame> original_frame; - uint8_t data[10]; - std::fill_n(data, 10, 5); - std::span<uint8_t> data_view(data); - EXPECT_CALL(original_frame, GetData()).WillRepeatedly(Return(data_view)); - VideoFrameMetadata metadata; - std::vector<uint32_t> csrcs{123, 321}; - // Copy csrcs rather than moving so we can compare in an EXPECT_EQ later. - metadata.SetCsrcs(csrcs); - - EXPECT_CALL(original_frame, Metadata()).WillRepeatedly(Return(metadata)); - auto cloned_frame = CloneVideoFrame(&original_frame); - - EXPECT_EQ(cloned_frame->GetData().size(), 10u); - EXPECT_THAT(cloned_frame->GetData(), Each(5u)); - EXPECT_EQ(cloned_frame->Metadata().GetCsrcs(), csrcs); -} - -} // namespace -} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 54441ab..106003f 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc
@@ -17,6 +17,7 @@ #include <span> #include <string> #include <utility> +#include <variant> #include <vector> #include "api/frame_transformer_interface.h" @@ -237,9 +238,12 @@ auto* transformed_video_frame = static_cast<TransformableVideoSenderFrame*>(transformed_frame.get()); RTC_CHECK(transformed_video_frame->CaptureTime().has_value()); + RTC_CHECK(std::holds_alternative<RtpTimestampWithOffset>( + transformed_video_frame->GetRtpTimestampInfo())); sender_->SendVideo(transformed_video_frame->GetPayloadType(), transformed_video_frame->GetCodecType(), - transformed_video_frame->GetTimestamp(), + std::get<RtpTimestampWithOffset>( + transformed_video_frame->GetRtpTimestampInfo()), *transformed_video_frame->CaptureTime(), transformed_video_frame->GetData(), transformed_video_frame->GetPreTransformPayloadSize(), @@ -253,9 +257,12 @@ // TODO(bugs.webrtc.org/14708): Use an actual RTT estimate for the // retransmission time instead of a const default, in the same way as a // locally encoded frame. + RTC_CHECK(std::holds_alternative<RtpTimestampWithOffset>( + transformed_video_frame->GetRtpTimestampInfo())); sender_->SendVideo(transformed_video_frame->GetPayloadType(), metadata.GetCodec(), - transformed_video_frame->GetTimestamp(), + std::get<RtpTimestampWithOffset>( + transformed_video_frame->GetRtpTimestampInfo()), /*capture_time=*/Timestamp::MinusInfinity(), transformed_video_frame->GetData(), transformed_video_frame->GetData().size(), @@ -302,10 +309,12 @@ // locally encoded frame. VideoFrameMetadata metadata = original->Metadata(); RTPVideoHeader new_header = RTPVideoHeader::FromMetadata(metadata); + RTC_CHECK(std::holds_alternative<RtpTimestampWithOffset>( + original->GetRtpTimestampInfo())); return std::make_unique<TransformableVideoSenderFrame>( encoded_image, new_header, original->GetPayloadType(), new_header.codec, - original->GetTimestamp(), kDefaultRetransmissionsTime, - original->GetSsrc(), metadata.GetCsrcs(), + std::get<RtpTimestampWithOffset>(original->GetRtpTimestampInfo()), + kDefaultRetransmissionsTime, original->GetSsrc(), metadata.GetCsrcs(), original->Rid().value_or(std::string())); }
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc index 6238d09..47797bc 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc
@@ -304,7 +304,9 @@ ON_CALL(*mock_receiver_frame, GetData).WillByDefault(Return(buffer)); ON_CALL(*mock_receiver_frame, GetPayloadType) .WillByDefault(Return(payload_type)); - ON_CALL(*mock_receiver_frame, GetTimestamp).WillByDefault(Return(timestamp)); + ON_CALL(*mock_receiver_frame, GetRtpTimestampInfo) + .WillByDefault( + Return(RtpTimestampInfo{RtpTimestampWithOffset{timestamp}})); scoped_refptr<TransformedFrameCallback> callback; EXPECT_CALL(*frame_transformer_, RegisterTransformedFrameSinkCallback) @@ -341,10 +343,16 @@ auto& video_frame = static_cast<TransformableVideoFrameInterface&>(*frame); uint32_t rtp_timestamp = 12345; - ASSERT_FALSE(video_frame.GetTimestamp() == rtp_timestamp); + ASSERT_TRUE(std::holds_alternative<RtpTimestampWithOffset>( + video_frame.GetRtpTimestampInfo())); + ASSERT_FALSE(std::get<RtpTimestampWithOffset>( + video_frame.GetRtpTimestampInfo()) == rtp_timestamp); video_frame.SetRTPTimestamp(rtp_timestamp); - EXPECT_EQ(video_frame.GetTimestamp(), rtp_timestamp); + EXPECT_TRUE(std::holds_alternative<RtpTimestampWithOffset>( + video_frame.GetRtpTimestampInfo())); + EXPECT_EQ(std::get<RtpTimestampWithOffset>(video_frame.GetRtpTimestampInfo()), + rtp_timestamp); } TEST_F(RtpSenderVideoFrameTransformerDelegateTest,
diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc index 1b4ea9ec..9ce1abf 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc
@@ -16,6 +16,7 @@ #include <span> #include <string> #include <utility> +#include <variant> #include <vector> #include "absl/memory/memory.h" @@ -237,6 +238,8 @@ VideoSendTiming timing; std::span<const uint8_t> data = transformed_frame->GetData(); Timestamp receive_time = clock_->CurrentTime(); + RTC_CHECK(std::holds_alternative<RtpTimestampWithOffset>( + transformed_frame->GetRtpTimestampInfo())); receiver_->ManageFrame(std::make_unique<RtpFrameObject>( /*first_seq_num=*/metadata.GetFrameId().value_or(0), /*last_seq_num=*/metadata.GetFrameId().value_or(0), @@ -244,7 +247,9 @@ /*times_nacked=*/0, /*first_packet_received_time=*/receive_time, /*last_packet_received_time=*/receive_time, - /*rtp_timestamp=*/transformed_frame->GetTimestamp(), + /*rtp_timestamp=*/ + std::get<RtpTimestampWithOffset>( + transformed_frame->GetRtpTimestampInfo()), /*ntp_time_ms=*/0, timing, transformed_frame->GetPayloadType(), metadata.GetCodec(), metadata.GetRotation(), metadata.GetContentType(), video_header, video_header.color_space,
diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc index fde9c61..13017cf 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc
@@ -475,7 +475,6 @@ // Test the SetRTPTimestamp setter uint32_t new_timestamp = 112233u; frame->SetRTPTimestamp(new_timestamp); - EXPECT_EQ(frame->GetTimestamp(), new_timestamp); EXPECT_TRUE(std::holds_alternative<RtpTimestampWithOffset>( frame->GetRtpTimestampInfo())); EXPECT_EQ(