Insert frame transformer between Depacketizer and Decoder. Add a new API in RTReceiverInterface, to be called from the browser side to insert a frame transformer between the Depacketizer and the Decoder. The frame transformer is passed from RTReceiverInterface through the library to be eventually set in RtpVideoStreamReceiver, where the frame transformation will occur in the follow-up CL https://webrtc-review.googlesource.com/c/src/+/169130. This change is part of the implementation of the Insertable Streams Web API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md Design doc for WebRTC library changes: http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk Bug: webrtc:11380 Change-Id: I6b73cd16e3907e8b7709b852d6a2540ee11b4fed Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169129 Reviewed-by: Danil Chapovalov <danilchap@webrtc.org> Reviewed-by: Magnus Flodman <mflodman@webrtc.org> Reviewed-by: Harald Alvestrand <hta@webrtc.org> Reviewed-by: Karl Wiberg <kwiberg@webrtc.org> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org> Cr-Commit-Position: refs/heads/master@{#30654}
diff --git a/api/rtp_receiver_interface.cc b/api/rtp_receiver_interface.cc index d20516b..bc9aef5 100644 --- a/api/rtp_receiver_interface.cc +++ b/api/rtp_receiver_interface.cc
@@ -38,4 +38,7 @@ return nullptr; } +void RtpReceiverInterface::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {} + } // namespace webrtc
diff --git a/api/rtp_receiver_interface.h b/api/rtp_receiver_interface.h index 6052763..a15864e 100644 --- a/api/rtp_receiver_interface.h +++ b/api/rtp_receiver_interface.h
@@ -19,6 +19,7 @@ #include "api/crypto/frame_decryptor_interface.h" #include "api/dtls_transport_interface.h" +#include "api/frame_transformer_interface.h" #include "api/media_stream_interface.h" #include "api/media_types.h" #include "api/proxy.h" @@ -107,6 +108,12 @@ // user. This can be used to update the state of the object. virtual rtc::scoped_refptr<FrameDecryptorInterface> GetFrameDecryptor() const; + // Sets a frame transformer between the depacketizer and the decoder to enable + // client code to transform received frames according to their own processing + // logic. + virtual void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); + protected: ~RtpReceiverInterface() override = default; }; @@ -132,6 +139,9 @@ rtc::scoped_refptr<FrameDecryptorInterface>) PROXY_CONSTMETHOD0(rtc::scoped_refptr<FrameDecryptorInterface>, GetFrameDecryptor) +PROXY_METHOD1(void, + SetDepacketizerToDecoderFrameTransformer, + rtc::scoped_refptr<FrameTransformerInterface>) END_PROXY_MAP() } // namespace webrtc
diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index 0f5e8e0..765d802 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h
@@ -21,6 +21,7 @@ #include "api/call/transport.h" #include "api/crypto/crypto_options.h" #include "api/crypto/frame_decryptor_interface.h" +#include "api/frame_transformer_interface.h" #include "api/rtp_headers.h" #include "api/rtp_parameters.h" #include "api/transport/rtp/rtp_source.h" @@ -262,6 +263,8 @@ // Per PeerConnection cryptography options. CryptoOptions crypto_options; + + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer; }; // Starts stream activity.
diff --git a/media/base/media_channel.cc b/media/base/media_channel.cc index 29cf550..3417924 100644 --- a/media/base/media_channel.cc +++ b/media/base/media_channel.cc
@@ -52,6 +52,9 @@ void MediaChannel::SetEncoderToPacketizerFrameTransformer( uint32_t ssrc, rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {} +void MediaChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {} MediaSenderInfo::MediaSenderInfo() = default; MediaSenderInfo::~MediaSenderInfo() = default;
diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 9631722..8ee4a23 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h
@@ -291,6 +291,9 @@ virtual void SetEncoderToPacketizerFrameTransformer( uint32_t ssrc, rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer); + virtual void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer); protected: bool DscpEnabled() const { return enable_dscp_; }
diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 3f43f79..0c23ff8 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc
@@ -2941,6 +2941,13 @@ } } +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + frame_transformer) { + config_.frame_transformer = frame_transformer; +} + WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings() : flexfec_payload_type(-1), rtx_payload_type(-1) {} @@ -3152,6 +3159,17 @@ } } +void WebRtcVideoChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto matching_stream = receive_streams_.find(ssrc); + if (matching_stream != receive_streams_.end()) { + matching_stream->second->SetDepacketizerToDecoderFrameTransformer( + std::move(frame_transformer)); + } +} + // TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of // EncoderStreamFactory and instead set this value individually for each stream // in the VideoEncoderConfig.simulcast_layers.
diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index b1cbd1b..418576f 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h
@@ -228,6 +228,10 @@ uint32_t ssrc, rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) override; + void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override; private: class WebRtcVideoReceiveStream; @@ -463,6 +467,10 @@ void ClearRecordableEncodedFrameCallback(); void GenerateKeyFrame(); + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + frame_transformer); + private: void RecreateWebRtcVideoStream(); void MaybeRecreateWebRtcFlexfecStream();
diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index c6fb543..a63a0f6 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc
@@ -104,6 +104,18 @@ return frame_decryptor_; } +void VideoRtpReceiver::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) { + worker_thread_->Invoke<void>(RTC_FROM_HERE, [&] { + RTC_DCHECK_RUN_ON(worker_thread_); + frame_transformer_ = std::move(frame_transformer); + if (media_channel_ && ssrc_.has_value() && !stopped_) { + media_channel_->SetDepacketizerToDecoderFrameTransformer( + *ssrc_, frame_transformer_); + } + }); +} + void VideoRtpReceiver::Stop() { // TODO(deadbeef): Need to do more here to fully stop receiving packets. if (stopped_) { @@ -144,6 +156,11 @@ if (encoded_sink_enabled) { SetEncodedSinkEnabled(true); } + + if (frame_transformer_ && media_channel_ && ssrc_.has_value()) { + media_channel_->SetDepacketizerToDecoderFrameTransformer( + *ssrc_, frame_transformer_); + } }); // Attach any existing frame decryptor to the media channel.
diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index 0b8a73d..f66a8a7 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h
@@ -18,6 +18,7 @@ #include "absl/types/optional.h" #include "api/crypto/frame_decryptor_interface.h" +#include "api/frame_transformer_interface.h" #include "api/media_stream_interface.h" #include "api/media_types.h" #include "api/rtp_parameters.h" @@ -83,6 +84,9 @@ rtc::scoped_refptr<FrameDecryptorInterface> GetFrameDecryptor() const override; + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override; + // RtpReceiverInternal implementation. void Stop() override; void SetupMediaChannel(uint32_t ssrc) override; @@ -134,6 +138,8 @@ int attachment_id_ = 0; rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor_; rtc::scoped_refptr<DtlsTransportInterface> dtls_transport_; + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_ + RTC_GUARDED_BY(worker_thread_); // Allows to thread safely change jitter buffer delay. Handles caching cases // if |SetJitterBufferMinimumDelay| is called before start. rtc::scoped_refptr<JitterBufferDelayInterface> delay_;
diff --git a/video/BUILD.gn b/video/BUILD.gn index 7bab757..2f7ec17 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn
@@ -54,6 +54,7 @@ ":frame_dumping_decoder", "../api:array_view", "../api:fec_controller_api", + "../api:frame_transformer_interface", "../api:libjingle_peerconnection_api", "../api:rtp_parameters", "../api:scoped_refptr",
diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 52f1014..5a6c627 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc
@@ -12,6 +12,7 @@ #include <algorithm> #include <limits> +#include <memory> #include <utility> #include <vector> @@ -194,7 +195,8 @@ NackSender* nack_sender, KeyFrameRequestSender* keyframe_request_sender, video_coding::OnCompleteFrameCallback* complete_frame_callback, - rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) + rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) : clock_(clock), config_(*config), packet_router_(packet_router), @@ -221,7 +223,8 @@ packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()), has_received_frame_(false), frames_decryptable_(false), - absolute_capture_time_receiver_(clock) { + absolute_capture_time_receiver_(clock), + frame_transformer_(frame_transformer) { constexpr bool remb_candidate = true; if (packet_router_) packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); @@ -283,6 +286,33 @@ } } +RtpVideoStreamReceiver::RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) + : RtpVideoStreamReceiver(clock, + transport, + rtt_stats, + packet_router, + config, + rtp_receive_statistics, + receive_stats_proxy, + process_thread, + nack_sender, + keyframe_request_sender, + complete_frame_callback, + frame_decryptor, + nullptr) {} + RtpVideoStreamReceiver::~RtpVideoStreamReceiver() { RTC_DCHECK(secondary_sinks_.empty());
diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h index c0271f4..98b324c 100644 --- a/video/rtp_video_stream_receiver.h +++ b/video/rtp_video_stream_receiver.h
@@ -86,6 +86,29 @@ // requests are sent via the internal RtpRtcp module. KeyFrameRequestSender* keyframe_request_sender, video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); + + // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies + // to use the new constructor. + RTC_DEPRECATED + RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + // The packet router is optional; if provided, the RtpRtcp module for this + // stream is registered as a candidate for sending REMB and transport + // feedback. + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + // The KeyFrameRequestSender is optional; if not provided, key frame + // requests are sent via the internal RtpRtcp module. + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor); ~RtpVideoStreamReceiver() override; @@ -346,6 +369,8 @@ RTC_GUARDED_BY(worker_task_checker_); int64_t last_completed_picture_id_ = 0; + + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_; }; } // namespace webrtc
diff --git a/video/rtp_video_stream_receiver_unittest.cc b/video/rtp_video_stream_receiver_unittest.cc index 44f544f..d7d02b0 100644 --- a/video/rtp_video_stream_receiver_unittest.cc +++ b/video/rtp_video_stream_receiver_unittest.cc
@@ -155,7 +155,7 @@ Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, rtp_receive_statistics_.get(), nullptr, process_thread_.get(), &mock_nack_sender_, &mock_key_frame_request_sender_, - &mock_on_complete_frame_callback_, nullptr); + &mock_on_complete_frame_callback_, nullptr, nullptr); } RTPVideoHeader GetDefaultH264VideoHeader() {
diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index 0a2d819..246daad 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc
@@ -215,7 +215,8 @@ this, // NackSender nullptr, // Use default KeyFrameRequestSender this, // OnCompleteFrameCallback - config_.frame_decryptor), + config_.frame_decryptor, + config_.frame_transformer), rtp_stream_sync_(this), max_wait_for_keyframe_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() .MaxWaitForKeyframeMs()