Add a clone method to the video frame transformer API.
This will clone an encoded video frame into a sender frame,
preserving metadata as much as possible.
Bug: webrtc:14708
Change-Id: I6f68d2ee65ef85c32cc3c142a41346b81ba73533
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/284701
Commit-Queue: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Guido Urdaneta <guidou@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38733}
diff --git a/api/BUILD.gn b/api/BUILD.gn
index c661dc5..798fa13 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -1466,3 +1466,19 @@
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
+
+rtc_library("frame_transformer_factory") {
+ visibility = [ "*" ]
+ sources = [
+ "frame_transformer_factory.cc",
+ "frame_transformer_factory.h",
+ ]
+ deps = [
+ ":frame_transformer_interface",
+ ":scoped_refptr",
+ "../modules/rtp_rtcp",
+ "../rtc_base:refcount",
+ "video:encoded_frame",
+ "video:video_frame_metadata",
+ ]
+}
diff --git a/api/frame_transformer_factory.cc b/api/frame_transformer_factory.cc
new file mode 100644
index 0000000..af08372
--- /dev/null
+++ b/api/frame_transformer_factory.cc
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/frame_transformer_factory.h"
+
+#include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h"
+
+namespace webrtc {
+
+std::unique_ptr<TransformableVideoFrameInterface> CreateVideoSenderFrame() {
+ RTC_CHECK_NOTREACHED();
+ return nullptr;
+}
+
+std::unique_ptr<TransformableVideoFrameInterface> CreateVideoReceiverFrame() {
+ RTC_CHECK_NOTREACHED();
+ return nullptr;
+}
+
+std::unique_ptr<TransformableVideoFrameInterface> CloneVideoFrame(
+ TransformableVideoFrameInterface* original) {
+ // At the moment, only making sender frames from receiver frames is supported.
+ return CloneSenderVideoFrame(original);
+}
+
+} // namespace webrtc
diff --git a/api/frame_transformer_factory.h b/api/frame_transformer_factory.h
new file mode 100644
index 0000000..8ba9c29
--- /dev/null
+++ b/api/frame_transformer_factory.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_FRAME_TRANSFORMER_FACTORY_H_
+#define API_FRAME_TRANSFORMER_FACTORY_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/frame_transformer_interface.h"
+#include "api/scoped_refptr.h"
+#include "api/video/encoded_frame.h"
+#include "api/video/video_frame_metadata.h"
+
+// This file contains EXPERIMENTAL functions to create video frames from
+// either an old video frame or directly from parameters.
+// These functions will be used in Chrome functionality to manipulate
+// encoded frames from Javascript.
+namespace webrtc {
+
+// TODO(bugs.webrtc.org/14708): Add the required parameters to these APIs.
+std::unique_ptr<TransformableVideoFrameInterface> CreateVideoSenderFrame();
+// TODO(bugs.webrtc.org/14708): Consider whether Receiver frames ever make sense
+// to create.
+std::unique_ptr<TransformableVideoFrameInterface> CreateVideoReceiverFrame();
+// Creates a new frame with the same metadata as the original.
+// The original can be a sender or receiver frame.
+RTC_EXPORT std::unique_ptr<TransformableVideoFrameInterface> CloneVideoFrame(
+ TransformableVideoFrameInterface* original);
+} // namespace webrtc
+
+#endif // API_FRAME_TRANSFORMER_FACTORY_H_
diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn
index ead114b..e046882 100644
--- a/modules/rtp_rtcp/BUILD.gn
+++ b/modules/rtp_rtcp/BUILD.gn
@@ -596,6 +596,7 @@
]
deps = [
":fec_test_helper",
+ ":frame_transformer_factory_unittest",
":mock_rtp_rtcp",
":rtcp_transceiver",
":rtp_packetizer_av1_test_helper",
@@ -605,6 +606,7 @@
"../../api:array_view",
"../../api:create_time_controller",
"../../api:field_trials_registry",
+ "../../api:frame_transformer_factory",
"../../api:libjingle_peerconnection_api",
"../../api:mock_frame_encryptor",
"../../api:rtp_headers",
@@ -669,3 +671,19 @@
]
}
}
+
+rtc_source_set("frame_transformer_factory_unittest") {
+ testonly = true
+ sources = [ "source/frame_transformer_factory_unittest.cc" ]
+ deps = [
+ "../../api:frame_transformer_factory",
+ "../../api:transport_api",
+ "../../call:video_stream_api",
+ "../../modules/rtp_rtcp",
+ "../../rtc_base:rtc_event",
+ "../../test:mock_frame_transformer",
+ "../../test:test_support",
+ "../../video",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/memory" ]
+}
diff --git a/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc b/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc
new file mode 100644
index 0000000..e011a76
--- /dev/null
+++ b/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/frame_transformer_factory.h"
+
+#include <cstdio>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "api/call/transport.h"
+#include "call/video_receive_stream.h"
+#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
+#include "rtc_base/event.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/mock_frame_transformer.h"
+
+namespace webrtc {
+namespace {
+
+using testing::NiceMock;
+using testing::Return;
+
+class MockTransformableVideoFrame
+ : public webrtc::TransformableVideoFrameInterface {
+ public:
+ MOCK_METHOD(rtc::ArrayView<const uint8_t>, GetData, (), (const override));
+ MOCK_METHOD(void, SetData, (rtc::ArrayView<const uint8_t> data), (override));
+ MOCK_METHOD(uint8_t, GetPayloadType, (), (const, override));
+ MOCK_METHOD(uint32_t, GetSsrc, (), (const, override));
+ MOCK_METHOD(uint32_t, GetTimestamp, (), (const, override));
+ MOCK_METHOD(TransformableFrameInterface::Direction,
+ GetDirection,
+ (),
+ (const, override));
+ MOCK_METHOD(bool, IsKeyFrame, (), (const, override));
+ MOCK_METHOD(std::vector<uint8_t>, GetAdditionalData, (), (const, override));
+ MOCK_METHOD(const webrtc::VideoFrameMetadata&,
+ GetMetadata,
+ (),
+ (const, override));
+};
+
+TEST(FrameTransformerFactory, CloneVideoFrame) {
+ NiceMock<MockTransformableVideoFrame> original_frame;
+ uint8_t data[10];
+ std::fill_n(data, 10, 5);
+ rtc::ArrayView<uint8_t> data_view(data);
+ EXPECT_CALL(original_frame, GetData()).WillRepeatedly(Return(data_view));
+ auto cloned_frame = CloneVideoFrame(&original_frame);
+ EXPECT_EQ(cloned_frame->GetData().size(), 10u);
+ EXPECT_THAT(cloned_frame->GetData(), testing::Each(5u));
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc
index 3e42781..a60c393 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc
@@ -18,6 +18,7 @@
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "modules/rtp_rtcp/source/rtp_sender_video.h"
#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
namespace webrtc {
namespace {
@@ -161,8 +162,9 @@
EnsureEncoderQueueCreated();
- if (!sender_)
+ if (!sender_) {
return;
+ }
rtc::scoped_refptr<RTPSenderVideoFrameTransformerDelegate> delegate(this);
encoder_queue_->PostTask(
[delegate = std::move(delegate), frame = std::move(frame)]() mutable {
@@ -212,4 +214,37 @@
sender_ = nullptr;
}
}
+
+std::unique_ptr<TransformableVideoFrameInterface> CloneSenderVideoFrame(
+ TransformableVideoFrameInterface* original) {
+ auto encoded_image_buffer = EncodedImageBuffer::Create(
+ original->GetData().data(), original->GetData().size());
+ EncodedImage encoded_image;
+ encoded_image.SetEncodedData(encoded_image_buffer);
+ RTPVideoHeader new_header;
+ absl::optional<VideoCodecType> new_codec_type;
+ // TODO(bugs.webrtc.org/14708): Figure out a way to get the header information
+ // without casting to TransformableVideoSenderFrame.
+ if (original->GetDirection() ==
+ TransformableFrameInterface::Direction::kSender) {
+ // TODO(bugs.webrtc.org/14708): Figure out a way to bulletproof this cast.
+ auto original_as_sender =
+ static_cast<TransformableVideoSenderFrame*>(original);
+ new_header = original_as_sender->GetHeader();
+ new_codec_type = original_as_sender->GetCodecType();
+ } else {
+ // TODO(bugs.webrtc.org/14708): Make this codec dependent
+ new_header.video_type_header.emplace<RTPVideoHeaderVP8>();
+ new_codec_type = kVideoCodecVP8;
+ // TODO(bugs.webrtc.org/14708): Fill in the new_header when it's not
+ // `Direction::kSender`
+ }
+ // TODO(bugs.webrtc.org/14708): Fill in other EncodedImage parameters
+ return std::make_unique<TransformableVideoSenderFrame>(
+ encoded_image, new_header, original->GetPayloadType(), new_codec_type,
+ original->GetTimestamp(),
+ absl::nullopt, // expected_retransmission_time_ms
+ original->GetSsrc());
+}
+
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h
index 0c2a643..04cdfd3 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h
@@ -87,6 +87,10 @@
std::unique_ptr<TaskQueueBase, TaskQueueDeleter> owned_encoder_queue_;
};
+// Method to support cloning a Sender frame from another frame
+std::unique_ptr<TransformableVideoFrameInterface> CloneSenderVideoFrame(
+ TransformableVideoFrameInterface* original);
+
} // namespace webrtc
#endif // MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_FRAME_TRANSFORMER_DELEGATE_H_
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
index 1352712..3eacda8 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
@@ -17,6 +17,7 @@
#include "absl/memory/memory.h"
#include "api/field_trials_registry.h"
+#include "api/frame_transformer_factory.h"
#include "api/rtp_headers.h"
#include "api/task_queue/task_queue_base.h"
#include "api/task_queue/task_queue_factory.h"
@@ -38,6 +39,7 @@
#include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
#include "rtc_base/rate_limiter.h"
#include "rtc_base/thread.h"
#include "test/gmock.h"
@@ -1645,5 +1647,45 @@
kDefaultExpectedRetransmissionTimeMs);
}
+TEST_F(RtpSenderVideoWithFrameTransformerTest,
+ OnTransformedFrameSendsVideoWhenCloned) {
+ auto mock_frame_transformer =
+ rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
+ rtc::scoped_refptr<TransformedFrameCallback> callback;
+ EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback)
+ .WillOnce(SaveArg<0>(&callback));
+ std::unique_ptr<RTPSenderVideo> rtp_sender_video =
+ CreateSenderWithFrameTransformer(mock_frame_transformer);
+ ASSERT_TRUE(callback);
+
+ auto encoded_image = CreateDefaultEncodedImage();
+ RTPVideoHeader video_header;
+ video_header.frame_type = VideoFrameType::kVideoFrameKey;
+ ON_CALL(*mock_frame_transformer, Transform)
+ .WillByDefault(
+ [&callback](std::unique_ptr<TransformableFrameInterface> frame) {
+ auto clone = CloneVideoFrame(
+ static_cast<TransformableVideoFrameInterface*>(frame.get()));
+ EXPECT_TRUE(clone);
+ callback->OnTransformedFrame(std::move(clone));
+ });
+ auto encoder_queue = time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
+ "encoder_queue", TaskQueueFactory::Priority::NORMAL);
+ encoder_queue->PostTask([&] {
+ rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
+ *encoded_image, video_header,
+ kDefaultExpectedRetransmissionTimeMs);
+ });
+ time_controller_.AdvanceTime(TimeDelta::Zero());
+ EXPECT_EQ(transport_.packets_sent(), 1);
+ encoder_queue->PostTask([&] {
+ rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
+ *encoded_image, video_header,
+ kDefaultExpectedRetransmissionTimeMs);
+ });
+ time_controller_.AdvanceTime(TimeDelta::Zero());
+ EXPECT_EQ(transport_.packets_sent(), 2);
+}
+
} // namespace
} // namespace webrtc
