Deliver packet info to source listeners when audio playout is disabled.
Normally, packet/frame info is delivered to AudioReceiveStream's
source_tracker_ when an audio frame is pulled out of the stream (as a
side-effect of GetAudioFrameWithInfo). When playout is muted, though,
packets are thrown away in ChannelReceive::OnReceivedPayloadData, so
AudioRtpReceiver stops seeing updates to its RtpSources and any related
information (e.g. CSRCs and associated timestamps, levels).
Skipping the playout path here has a downside of being misaligned with
whatever playout delay would normally be, but it allows clients that
want to consume RtpSource information to be able to do so while playout
is muted.
Bug: None
Change-Id: Id00566b645de4196c2341611cd9e8b94b35aa157
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/203500
Reviewed-by: Henrik Lundin <henrik.lundin@webrtc.org>
Reviewed-by: Jakob Ivarsson <jakobi@webrtc.org>
Reviewed-by: Noah Richards <noahric@chromium.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Commit-Queue: Ranveer Aggarwal <ranvr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33236}
diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc
index 03dd4c0..e037ddc 100644
--- a/audio/audio_receive_stream.cc
+++ b/audio/audio_receive_stream.cc
@@ -24,6 +24,7 @@
#include "audio/conversion.h"
#include "call/rtp_config.h"
#include "call/rtp_stream_receiver_controller_interface.h"
+#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
@@ -118,8 +119,8 @@
webrtc::RtcEventLog* event_log,
std::unique_ptr<voe::ChannelReceiveInterface> channel_receive)
: audio_state_(audio_state),
- channel_receive_(std::move(channel_receive)),
- source_tracker_(clock) {
+ source_tracker_(clock),
+ channel_receive_(std::move(channel_receive)) {
RTC_LOG(LS_INFO) << "AudioReceiveStream: " << config.rtp.remote_ssrc;
RTC_DCHECK(config.decoder_factory);
RTC_DCHECK(config.rtcp_send_transport);
@@ -133,6 +134,11 @@
// Configure bandwidth estimation.
channel_receive_->RegisterReceiverCongestionControlObjects(packet_router);
+ // When output is muted, ChannelReceive will directly notify the source
+ // tracker of "delivered" frames, so RtpReceiver information will continue to
+ // be updated.
+ channel_receive_->SetSourceTracker(&source_tracker_);
+
// Register with transport.
rtp_stream_receiver_ = receiver_controller->CreateReceiver(
config.rtp.remote_ssrc, channel_receive_.get());
diff --git a/audio/audio_receive_stream.h b/audio/audio_receive_stream.h
index 0ccd21e..545a120 100644
--- a/audio/audio_receive_stream.h
+++ b/audio/audio_receive_stream.h
@@ -112,8 +112,8 @@
SequenceChecker module_process_thread_checker_;
webrtc::AudioReceiveStream::Config config_;
rtc::scoped_refptr<webrtc::AudioState> audio_state_;
- const std::unique_ptr<voe::ChannelReceiveInterface> channel_receive_;
SourceTracker source_tracker_;
+ const std::unique_ptr<voe::ChannelReceiveInterface> channel_receive_;
AudioSendStream* associated_send_stream_ = nullptr;
bool playing_ RTC_GUARDED_BY(worker_thread_checker_) = false;
diff --git a/audio/audio_receive_stream_unittest.cc b/audio/audio_receive_stream_unittest.cc
index fcd691e..99e3a56 100644
--- a/audio/audio_receive_stream_unittest.cc
+++ b/audio/audio_receive_stream_unittest.cc
@@ -106,6 +106,7 @@
}));
EXPECT_CALL(*channel_receive_, SetDepacketizerToDecoderFrameTransformer(_))
.Times(1);
+ EXPECT_CALL(*channel_receive_, SetSourceTracker(_));
stream_config_.rtp.local_ssrc = kLocalSsrc;
stream_config_.rtp.remote_ssrc = kRemoteSsrc;
diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc
index 1316ccb..8b6c9cf 100644
--- a/audio/channel_receive.cc
+++ b/audio/channel_receive.cc
@@ -162,6 +162,8 @@
int PreferredSampleRate() const override;
+ void SetSourceTracker(SourceTracker* source_tracker) override;
+
// Associate to a send channel.
// Used for obtaining RTT for a receive-only channel.
void SetAssociatedSendChannel(const ChannelSendInterface* channel) override;
@@ -219,6 +221,7 @@
std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
const uint32_t remote_ssrc_;
+ SourceTracker* source_tracker_ = nullptr;
// Info for GetSyncInfo is updated on network or worker thread, and queried on
// the worker thread.
@@ -233,6 +236,7 @@
AudioSinkInterface* audio_sink_ = nullptr;
AudioLevel _outputAudioLevel;
+ Clock* const clock_;
RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_);
// Timestamp of the audio pulled from NetEq.
@@ -287,6 +291,21 @@
if (!Playing()) {
// Avoid inserting into NetEQ when we are not playing. Count the
// packet as discarded.
+
+ // If we have a source_tracker_, tell it that the frame has been
+ // "delivered". Normally, this happens in AudioReceiveStream when audio
+ // frames are pulled out, but when playout is muted, nothing is pulling
+ // frames. The downside of this approach is that frames delivered this way
+ // won't be delayed for playout, and therefore will be unsynchronized with
+ // (a) audio delay when playing and (b) any audio/video synchronization. But
+ // the alternative is that muting playout also stops the SourceTracker from
+ // updating RtpSource information.
+ if (source_tracker_) {
+ RtpPacketInfos::vector_type packet_vector = {
+ RtpPacketInfo(rtpHeader, clock_->TimeInMilliseconds())};
+ source_tracker_->OnFrameDelivered(RtpPacketInfos(packet_vector));
+ }
+
return;
}
@@ -442,6 +461,10 @@
acm_receiver_.last_output_sample_rate_hz());
}
+void ChannelReceive::SetSourceTracker(SourceTracker* source_tracker) {
+ source_tracker_ = source_tracker;
+}
+
ChannelReceive::ChannelReceive(
Clock* clock,
ProcessThread* module_process_thread,
@@ -469,6 +492,7 @@
jitter_buffer_max_packets,
jitter_buffer_fast_playout)),
_outputAudioLevel(),
+ clock_(clock),
ntp_estimator_(clock),
playout_timestamp_rtp_(0),
playout_delay_ms_(0),
diff --git a/audio/channel_receive.h b/audio/channel_receive.h
index eef2db4..32bc813 100644
--- a/audio/channel_receive.h
+++ b/audio/channel_receive.h
@@ -28,6 +28,7 @@
#include "call/rtp_packet_sink_interface.h"
#include "call/syncable.h"
#include "modules/audio_coding/include/audio_coding_module_typedefs.h"
+#include "modules/rtp_rtcp/source/source_tracker.h"
#include "system_wrappers/include/clock.h"
// TODO(solenberg, nisse): This file contains a few NOLINT marks, to silence
@@ -135,6 +136,10 @@
virtual int PreferredSampleRate() const = 0;
+ // Sets the source tracker to notify about "delivered" packets when output is
+ // muted.
+ virtual void SetSourceTracker(SourceTracker* source_tracker) = 0;
+
// Associate to a send channel.
// Used for obtaining RTT for a receive-only channel.
virtual void SetAssociatedSendChannel(
diff --git a/audio/mock_voe_channel_proxy.h b/audio/mock_voe_channel_proxy.h
index 52e5b2f..7f140d4 100644
--- a/audio/mock_voe_channel_proxy.h
+++ b/audio/mock_voe_channel_proxy.h
@@ -59,6 +59,7 @@
(int sample_rate_hz, AudioFrame*),
(override));
MOCK_METHOD(int, PreferredSampleRate, (), (const, override));
+ MOCK_METHOD(void, SetSourceTracker, (SourceTracker*), (override));
MOCK_METHOD(void,
SetAssociatedSendChannel,
(const voe::ChannelSendInterface*),
