Support shortcircuiting encoded transforms
Add a StartShortCircuiting() callback to allow clients which have
configured Encoded Transforms when creating a PeerConnection to have
all frames skip the transform. This offers a zero cost path for streams
which don't need transforms.
This is preferable to uninstalling/not installing the transform to allow
implementing the behaviour in
https://w3c.github.io/webrtc-encoded-transform/#stream-creation -
giving web apps a chance to configure transforms within a short window
(before the next JS event loop run, so usually sub-millisecond) after stream creation, without any untransformed frames passing.
Usage in Chromium: crrev.com/c/5040731
Bug: chromium:1502781
Change-Id: I803477db1df51e80bdedf6c84d2d3695b088de83
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/327601
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Tony Herre <herre@google.com>
Cr-Commit-Position: refs/heads/main@{#41184}
diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h
index 94f8dab..5bbcffe 100644
--- a/api/frame_transformer_interface.h
+++ b/api/frame_transformer_interface.h
@@ -94,6 +94,12 @@
virtual void OnTransformedFrame(
std::unique_ptr<TransformableFrameInterface> frame) = 0;
+ // Request to no longer be called on each frame, instead having frames be
+ // sent directly to OnTransformedFrame without additional work.
+ // TODO(crbug.com/1502781): Make pure virtual once all mocks have
+ // implementations.
+ virtual void StartShortCircuiting() {}
+
protected:
~TransformedFrameCallback() override = default;
};
diff --git a/audio/channel_receive_frame_transformer_delegate.cc b/audio/channel_receive_frame_transformer_delegate.cc
index e87566b..3cad530 100644
--- a/audio/channel_receive_frame_transformer_delegate.cc
+++ b/audio/channel_receive_frame_transformer_delegate.cc
@@ -100,9 +100,13 @@
uint32_t ssrc,
const std::string& codec_mime_type) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
- frame_transformer_->Transform(
- std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc,
- codec_mime_type));
+ if (short_circuit_) {
+ receive_frame_callback_(packet, header);
+ } else {
+ frame_transformer_->Transform(
+ std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc,
+ codec_mime_type));
+ }
}
void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame(
@@ -114,6 +118,14 @@
});
}
+void ChannelReceiveFrameTransformerDelegate::StartShortCircuiting() {
+ rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate(this);
+ channel_receive_thread_->PostTask([delegate = std::move(delegate)]() mutable {
+ RTC_DCHECK_RUN_ON(&delegate->sequence_checker_);
+ delegate->short_circuit_ = true;
+ });
+}
+
void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
std::unique_ptr<TransformableFrameInterface> frame) const {
RTC_DCHECK_RUN_ON(&sequence_checker_);
diff --git a/audio/channel_receive_frame_transformer_delegate.h b/audio/channel_receive_frame_transformer_delegate.h
index 97bcacd..ac7cef0 100644
--- a/audio/channel_receive_frame_transformer_delegate.h
+++ b/audio/channel_receive_frame_transformer_delegate.h
@@ -56,6 +56,8 @@
void OnTransformedFrame(
std::unique_ptr<TransformableFrameInterface> frame) override;
+ void StartShortCircuiting() override;
+
// Delegates the call to ChannelReceive::OnReceivedPayloadData on the
// `channel_receive_thread_`, by calling `receive_frame_callback_`.
void ReceiveFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
@@ -70,6 +72,7 @@
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
RTC_GUARDED_BY(sequence_checker_);
TaskQueueBase* const channel_receive_thread_;
+ bool short_circuit_ RTC_GUARDED_BY(sequence_checker_) = false;
};
} // namespace webrtc
diff --git a/audio/channel_receive_frame_transformer_delegate_unittest.cc b/audio/channel_receive_frame_transformer_delegate_unittest.cc
index 9e655cc..8bdf217 100644
--- a/audio/channel_receive_frame_transformer_delegate_unittest.cc
+++ b/audio/channel_receive_frame_transformer_delegate_unittest.cc
@@ -150,5 +150,29 @@
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
+TEST(ChannelReceiveFrameTransformerDelegateTest,
+ ShortCircuitingSkipsTransform) {
+ rtc::AutoThread main_thread;
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>();
+ MockChannelReceive mock_channel;
+ rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate =
+ rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>(
+ mock_channel.callback(), mock_frame_transformer,
+ rtc::Thread::Current());
+ const uint8_t data[] = {1, 2, 3, 4};
+ rtc::ArrayView<const uint8_t> packet(data, sizeof(data));
+ RTPHeader header;
+
+ delegate->StartShortCircuiting();
+ rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
+
+ // Will not call the actual transformer.
+ EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
+ // Will pass the frame straight to the channel.
+ EXPECT_CALL(mock_channel, ReceiveFrame);
+ delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
+}
+
} // namespace
} // namespace webrtc
diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc
index 4e8d9fe..f66f29d 100644
--- a/audio/channel_send_frame_transformer_delegate.cc
+++ b/audio/channel_send_frame_transformer_delegate.cc
@@ -137,6 +137,16 @@
int64_t absolute_capture_timestamp_ms,
uint32_t ssrc,
const std::string& codec_mimetype) {
+ {
+ MutexLock lock(&send_lock_);
+ if (short_circuit_) {
+ send_frame_callback_(
+ frame_type, payload_type, rtp_timestamp,
+ rtc::ArrayView<const uint8_t>(payload_data, payload_size),
+ absolute_capture_timestamp_ms);
+ return;
+ }
+ }
frame_transformer_->Transform(
std::make_unique<TransformableOutgoingAudioFrame>(
frame_type, payload_type, rtp_timestamp, payload_data, payload_size,
@@ -155,6 +165,11 @@
});
}
+void ChannelSendFrameTransformerDelegate::StartShortCircuiting() {
+ MutexLock lock(&send_lock_);
+ short_circuit_ = true;
+}
+
void ChannelSendFrameTransformerDelegate::SendFrame(
std::unique_ptr<TransformableFrameInterface> frame) const {
MutexLock lock(&send_lock_);
diff --git a/audio/channel_send_frame_transformer_delegate.h b/audio/channel_send_frame_transformer_delegate.h
index 2306dfc..bcec018 100644
--- a/audio/channel_send_frame_transformer_delegate.h
+++ b/audio/channel_send_frame_transformer_delegate.h
@@ -65,6 +65,8 @@
void OnTransformedFrame(
std::unique_ptr<TransformableFrameInterface> frame) override;
+ void StartShortCircuiting() override;
+
// Delegates the call to ChannelSend::SendRtpAudio on the `encoder_queue_`,
// by calling `send_audio_callback_`.
void SendFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
@@ -77,6 +79,7 @@
SendFrameCallback send_frame_callback_ RTC_GUARDED_BY(send_lock_);
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_);
+ bool short_circuit_ RTC_GUARDED_BY(send_lock_) = false;
};
std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame(
diff --git a/audio/channel_send_frame_transformer_delegate_unittest.cc b/audio/channel_send_frame_transformer_delegate_unittest.cc
index 0c8e387..a47cbd5 100644
--- a/audio/channel_send_frame_transformer_delegate_unittest.cc
+++ b/audio/channel_send_frame_transformer_delegate_unittest.cc
@@ -168,5 +168,25 @@
channel_queue.WaitForPreviouslyPostedTasks();
}
+TEST(ChannelSendFrameTransformerDelegateTest, ShortCircuitingSkipsTransform) {
+ TaskQueueForTest channel_queue("channel_queue");
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>();
+ MockChannelSend mock_channel;
+ rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate =
+ rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>(
+ mock_channel.callback(), mock_frame_transformer, &channel_queue);
+
+ delegate->StartShortCircuiting();
+
+ // Will not call the actual transformer.
+ EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
+ // Will pass the frame straight to the channel.
+ EXPECT_CALL(mock_channel, SendFrame);
+ const uint8_t data[] = {1, 2, 3, 4};
+ delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0,
+ /*ssrc=*/0, /*mimeType=*/"audio/opus");
+}
+
} // namespace
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc
index d255ef4..26152df 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc
@@ -18,6 +18,7 @@
#include "api/task_queue/task_queue_factory.h"
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
namespace webrtc {
namespace {
@@ -147,6 +148,17 @@
const EncodedImage& encoded_image,
RTPVideoHeader video_header,
TimeDelta expected_retransmission_time) {
+ {
+ MutexLock lock(&sender_lock_);
+ if (short_circuit_) {
+ sender_->SendVideo(payload_type, codec_type, rtp_timestamp,
+ encoded_image.CaptureTime(),
+ *encoded_image.GetEncodedData(), encoded_image.size(),
+ video_header, expected_retransmission_time,
+ /*csrcs=*/{});
+ return true;
+ }
+ }
frame_transformer_->Transform(std::make_unique<TransformableVideoSenderFrame>(
encoded_image, video_header, payload_type, codec_type, rtp_timestamp,
expected_retransmission_time, ssrc_,
@@ -169,6 +181,11 @@
});
}
+void RTPSenderVideoFrameTransformerDelegate::StartShortCircuiting() {
+ MutexLock lock(&sender_lock_);
+ short_circuit_ = true;
+}
+
void RTPSenderVideoFrameTransformerDelegate::SendVideo(
std::unique_ptr<TransformableFrameInterface> transformed_frame) const {
RTC_DCHECK_RUN_ON(transformation_queue_.get());
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h
index a333db2..243f22c 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h
@@ -75,6 +75,8 @@
void OnTransformedFrame(
std::unique_ptr<TransformableFrameInterface> frame) override;
+ void StartShortCircuiting() override;
+
// Delegates the call to RTPSendVideo::SendVideo on the `encoder_queue_`.
void SendVideo(std::unique_ptr<TransformableFrameInterface> frame) const
RTC_RUN_ON(transformation_queue_);
@@ -107,6 +109,7 @@
// Used when the encoded frames arrives without a current task queue. This can
// happen if a hardware encoder was used.
std::unique_ptr<TaskQueueBase, TaskQueueDeleter> transformation_queue_;
+ bool short_circuit_ RTC_GUARDED_BY(sender_lock_) = false;
};
// Method to support cloning a Sender frame from another frame
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc
index a376be7..ad00e64 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc
@@ -289,5 +289,29 @@
EXPECT_EQ(video_frame.GetTimestamp(), rtp_timestamp);
}
+TEST_F(RtpSenderVideoFrameTransformerDelegateTest,
+ ShortCircuitingSkipsTransform) {
+ auto delegate = rtc::make_ref_counted<RTPSenderVideoFrameTransformerDelegate>(
+ &test_sender_, frame_transformer_,
+ /*ssrc=*/1111, time_controller_.CreateTaskQueueFactory().get());
+ EXPECT_CALL(*frame_transformer_,
+ RegisterTransformedFrameSinkCallback(_, 1111));
+ delegate->Init();
+
+ delegate->StartShortCircuiting();
+
+ // Will not call the actual transformer.
+ EXPECT_CALL(*frame_transformer_, Transform).Times(0);
+ // Will pass the frame straight to the reciever.
+ EXPECT_CALL(test_sender_, SendVideo);
+
+ EncodedImage encoded_image;
+ encoded_image.SetEncodedData(EncodedImageBuffer::Create(1));
+ delegate->TransformFrame(
+ /*payload_type=*/1, VideoCodecType::kVideoCodecVP8, /*rtp_timestamp=*/2,
+ encoded_image, RTPVideoHeader(),
+ /*expected_retransmission_time=*/TimeDelta::PlusInfinity());
+}
+
} // namespace
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc
index ddbd22e..fbd10c4 100644
--- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc
+++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc
@@ -17,6 +17,7 @@
#include "absl/memory/memory.h"
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
#include "rtc_base/thread.h"
namespace webrtc {
@@ -119,9 +120,14 @@
void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame(
std::unique_ptr<RtpFrameObject> frame) {
RTC_DCHECK_RUN_ON(&network_sequence_checker_);
- frame_transformer_->Transform(
- std::make_unique<TransformableVideoReceiverFrame>(std::move(frame), ssrc_,
- receiver_));
+ if (short_circuit_) {
+ // Just pass the frame straight back.
+ receiver_->ManageFrame(std::move(frame));
+ } else {
+ frame_transformer_->Transform(
+ std::make_unique<TransformableVideoReceiverFrame>(std::move(frame),
+ ssrc_, receiver_));
+ }
}
void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame(
@@ -134,6 +140,20 @@
});
}
+void RtpVideoStreamReceiverFrameTransformerDelegate::StartShortCircuiting() {
+ rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate(
+ this);
+ network_thread_->PostTask([delegate = std::move(delegate)]() mutable {
+ delegate->StartShortCircuitingOnNetworkSequence();
+ });
+}
+
+void RtpVideoStreamReceiverFrameTransformerDelegate::
+ StartShortCircuitingOnNetworkSequence() {
+ RTC_DCHECK_RUN_ON(&network_sequence_checker_);
+ short_circuit_ = true;
+}
+
void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
std::unique_ptr<TransformableFrameInterface> frame) {
RTC_DCHECK_RUN_ON(&network_sequence_checker_);
diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h
index 62a42fd..f08fc69 100644
--- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h
+++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h
@@ -55,6 +55,8 @@
void OnTransformedFrame(
std::unique_ptr<TransformableFrameInterface> frame) override;
+ void StartShortCircuiting() override;
+
// Delegates the call to RtpVideoFrameReceiver::ManageFrame on the
// `network_thread_`.
void ManageFrame(std::unique_ptr<TransformableFrameInterface> frame);
@@ -63,6 +65,8 @@
~RtpVideoStreamReceiverFrameTransformerDelegate() override = default;
private:
+ void StartShortCircuitingOnNetworkSequence();
+
RTC_NO_UNIQUE_ADDRESS SequenceChecker network_sequence_checker_;
RtpVideoFrameReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_);
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
@@ -70,6 +74,7 @@
rtc::Thread* const network_thread_;
const uint32_t ssrc_;
Clock* const clock_;
+ bool short_circuit_ RTC_GUARDED_BY(network_sequence_checker_) = false;
};
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc
index f403c91..cf30626 100644
--- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc
@@ -349,5 +349,28 @@
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
+TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
+ ShortCircuitingSkipsTransform) {
+ rtc::AutoThread main_thread_;
+ TestRtpVideoFrameReceiver receiver;
+ auto mock_frame_transformer =
+ rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
+ SimulatedClock clock(0);
+ auto delegate =
+ rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
+ &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
+ 1111);
+ delegate->Init();
+
+ delegate->StartShortCircuiting();
+ rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
+
+ // Will not call the actual transformer.
+ EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
+ // Will pass the frame straight to the reciever.
+ EXPECT_CALL(receiver, ManageFrame);
+ delegate->TransformFrame(CreateRtpFrameObject());
+}
+
} // namespace
} // namespace webrtc
