Google Git
Sign in
webrtc / src / 661ed679a2f6a8ad5e9153e610b6a61f4171bd97 / . / audio / channel_receive.cc
blob: ea4c912d2871ca189fac761f974eedefe956f9e3 [file] [log] [blame]
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "audio/channel_receive.h"
#include <algorithm>
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "api/audio/audio_device.h"
#include "api/crypto/frame_decryptor_interface.h"
#include "api/frame_transformer_interface.h"
#include "api/neteq/default_neteq_factory.h"
#include "api/neteq/neteq.h"
#include "api/rtc_event_log/rtc_event_log.h"
#include "api/sequence_checker.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/task_queue/task_queue_base.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "audio/audio_level.h"
#include "audio/channel_receive_frame_transformer_delegate.h"
#include "audio/channel_send.h"
#include "audio/utility/audio_frame_operations.h"
#include "logging/rtc_event_log/events/rtc_event_audio_playout.h"
#include "logging/rtc_event_log/events/rtc_event_neteq_set_minimum_delay.h"
#include "modules/audio_coding/acm2/acm_resampler.h"
#include "modules/audio_coding/acm2/call_statistics.h"
#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h"
#include "modules/pacing/packet_router.h"
#include "modules/rtp_rtcp/include/receive_statistics.h"
#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_minmax.h"
#include "rtc_base/numerics/sequence_number_unwrapper.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/time_utils.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/metrics.h"
#include "system_wrappers/include/ntp_time.h"
namespace webrtc {
namespace voe {
namespace {
constexpr double kAudioSampleDurationSeconds = 0.01;
// Video Sync.
constexpr int kVoiceEngineMinMinPlayoutDelayMs = 0;
constexpr int kVoiceEngineMaxMinPlayoutDelayMs = 10000;
std::unique_ptr<NetEq> CreateNetEq(
NetEqFactory* neteq_factory,
std::optional<AudioCodecPairId> codec_pair_id,
size_t jitter_buffer_max_packets,
bool jitter_buffer_fast_playout,
int jitter_buffer_min_delay_ms,
const Environment& env,
scoped_refptr<AudioDecoderFactory> decoder_factory) {
NetEq::Config config;
config.codec_pair_id = codec_pair_id;
config.max_packets_in_buffer = jitter_buffer_max_packets;
config.enable_fast_accelerate = jitter_buffer_fast_playout;
config.enable_muted_state = true;
config.min_delay_ms = jitter_buffer_min_delay_ms;
if (neteq_factory) {
return neteq_factory->Create(env, config, std::move(decoder_factory));
}
return DefaultNetEqFactory().Create(env, config, std::move(decoder_factory));
}
class ChannelReceive : public ChannelReceiveInterface,
public RtcpPacketTypeCounterObserver {
public:
// Used for receive streams.
ChannelReceive(
const Environment& env,
NetEqFactory* neteq_factory,
AudioDeviceModule* audio_device_module,
Transport* rtcp_send_transport,
uint32_t local_ssrc,
uint32_t remote_ssrc,
size_t jitter_buffer_max_packets,
bool jitter_buffer_fast_playout,
int jitter_buffer_min_delay_ms,
bool enable_non_sender_rtt,
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
std::optional<AudioCodecPairId> codec_pair_id,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
const webrtc::CryptoOptions& crypto_options,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
~ChannelReceive() override;
void SetSink(AudioSinkInterface* sink) override;
void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
// API methods
void StartPlayout() override;
void StopPlayout() override;
// Codecs
std::optional<std::pair<int, SdpAudioFormat>> GetReceiveCodec()
const override;
void ReceivedRTCPPacket(const uint8_t* data, size_t length) override;
// RtpPacketSinkInterface.
void OnRtpPacket(const RtpPacketReceived& packet) override;
// Muting, Volume and Level.
void SetChannelOutputVolumeScaling(float scaling) override;
int GetSpeechOutputLevelFullRange() const override;
// See description of "totalAudioEnergy" in the WebRTC stats spec:
// https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
double GetTotalOutputEnergy() const override;
double GetTotalOutputDuration() const override;
// Stats.
NetworkStatistics GetNetworkStatistics(
bool get_and_clear_legacy_stats) const override;
AudioDecodingCallStats GetDecodingCallStatistics() const override;
// Audio+Video Sync.
uint32_t GetDelayEstimate() const override;
bool SetMinimumPlayoutDelay(int delayMs) override;
bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp,
int64_t* time_ms) const override;
void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms,
int64_t time_ms) override;
std::optional<int64_t> GetCurrentEstimatedPlayoutNtpTimestampMs(
int64_t now_ms) const override;
// Audio quality.
bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override;
int GetBaseMinimumPlayoutDelayMs() const override;
// Produces the transport-related timestamps; current_delay_ms is left unset.
std::optional<Syncable::Info> GetSyncInfo() const override;
void RegisterReceiverCongestionControlObjects(
PacketRouter* packet_router) override;
void ResetReceiverCongestionControlObjects() override;
CallReceiveStatistics GetRTCPStatistics() const override;
void SetNACKStatus(bool enable, int maxNumberOfPackets) override;
void SetRtcpMode(webrtc::RtcpMode mode) override;
void SetNonSenderRttMeasurement(bool enabled) override;
AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo(
int sample_rate_hz,
AudioFrame* audio_frame) override;
int PreferredSampleRate() const override;
std::vector<RtpSource> GetSources() const override;
// Associate to a send channel.
// Used for obtaining RTT for a receive-only channel.
void SetAssociatedSendChannel(const ChannelSendInterface* channel) override;
// Sets a frame transformer between the depacketizer and the decoder, to
// transform the received frames before decoding them.
void SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
override;
void SetFrameDecryptor(rtc::scoped_refptr<webrtc::FrameDecryptorInterface>
frame_decryptor) override;
void OnLocalSsrcChange(uint32_t local_ssrc) override;
uint32_t GetLocalSsrc() const override;
void RtcpPacketTypesCounterUpdated(
uint32_t ssrc,
const RtcpPacketTypeCounter& packet_counter) override;
private:
void ReceivePacket(const uint8_t* packet,
size_t packet_length,
const RTPHeader& header,
Timestamp receive_time) RTC_RUN_ON(worker_thread_checker_);
int ResendPackets(const uint16_t* sequence_numbers, int length);
void UpdatePlayoutTimestamp(bool rtcp, int64_t now_ms)
RTC_RUN_ON(worker_thread_checker_);
int GetRtpTimestampRateHz() const;
void OnReceivedPayloadData(rtc::ArrayView<const uint8_t> payload,
const RTPHeader& rtpHeader,
Timestamp receive_time)
RTC_RUN_ON(worker_thread_checker_);
void InitFrameTransformerDelegate(
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
RTC_RUN_ON(worker_thread_checker_);
// Thread checkers document and lock usage of some methods to specific threads
// we know about. The goal is to eventually split up voe::ChannelReceive into
// parts with single-threaded semantics, and thereby reduce the need for
// locks.
RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_thread_checker_;
RTC_NO_UNIQUE_ADDRESS SequenceChecker network_thread_checker_;
const Environment env_;
TaskQueueBase* const worker_thread_;
ScopedTaskSafety worker_safety_;
// Methods accessed from audio and video threads are checked for sequential-
// only access. We don't necessarily own and control these threads, so thread
// checkers cannot be used. E.g. Chromium may transfer "ownership" from one
// audio thread to another, but access is still sequential.
rtc::RaceChecker audio_thread_race_checker_;
Mutex callback_mutex_;
Mutex volume_settings_mutex_;
mutable Mutex call_stats_mutex_;
bool playing_ RTC_GUARDED_BY(worker_thread_checker_) = false;
// Indexed by payload type.
std::map<uint8_t, int> payload_type_frequencies_;
std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
const uint32_t remote_ssrc_;
SourceTracker source_tracker_ RTC_GUARDED_BY(&worker_thread_checker_);
// Info for GetSyncInfo is updated on network or worker thread, and queried on
// the worker thread.
std::optional<uint32_t> last_received_rtp_timestamp_
RTC_GUARDED_BY(&worker_thread_checker_);
std::optional<int64_t> last_received_rtp_system_time_ms_
RTC_GUARDED_BY(&worker_thread_checker_);
const std::unique_ptr<NetEq> neteq_; // NetEq is thread-safe; no lock needed.
acm2::ResamplerHelper resampler_helper_
RTC_GUARDED_BY(audio_thread_race_checker_);
acm2::CallStatistics call_stats_ RTC_GUARDED_BY(call_stats_mutex_);
AudioSinkInterface* audio_sink_ = nullptr;
AudioLevel _outputAudioLevel;
RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_);
// Timestamp of the audio pulled from NetEq.
std::optional<uint32_t> jitter_buffer_playout_timestamp_;
uint32_t playout_timestamp_rtp_ RTC_GUARDED_BY(worker_thread_checker_);
std::optional<int64_t> playout_timestamp_rtp_time_ms_
RTC_GUARDED_BY(worker_thread_checker_);
uint32_t playout_delay_ms_ RTC_GUARDED_BY(worker_thread_checker_);
std::optional<int64_t> playout_timestamp_ntp_
RTC_GUARDED_BY(worker_thread_checker_);
std::optional<int64_t> playout_timestamp_ntp_time_ms_
RTC_GUARDED_BY(worker_thread_checker_);
mutable Mutex ts_stats_lock_;
webrtc::RtpTimestampUnwrapper rtp_ts_wraparound_handler_;
// The rtp timestamp of the first played out audio frame.
int64_t capture_start_rtp_time_stamp_;
// The capture ntp time (in local timebase) of the first played out audio
// frame.
int64_t capture_start_ntp_time_ms_ RTC_GUARDED_BY(ts_stats_lock_);
AudioDeviceModule* _audioDeviceModulePtr;
float _outputGain RTC_GUARDED_BY(volume_settings_mutex_);
const ChannelSendInterface* associated_send_channel_
RTC_GUARDED_BY(network_thread_checker_);
PacketRouter* packet_router_ = nullptr;
SequenceChecker construction_thread_;
// E2EE Audio Frame Decryption
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor_
RTC_GUARDED_BY(worker_thread_checker_);
webrtc::CryptoOptions crypto_options_;
webrtc::AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_
RTC_GUARDED_BY(worker_thread_checker_);
webrtc::CaptureClockOffsetUpdater capture_clock_offset_updater_
RTC_GUARDED_BY(ts_stats_lock_);
rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate>
frame_transformer_delegate_;
// Counter that's used to control the frequency of reporting histograms
// from the `GetAudioFrameWithInfo` callback.
int audio_frame_interval_count_ RTC_GUARDED_BY(audio_thread_race_checker_) =
0;
// Controls how many callbacks we let pass by before reporting callback stats.
// A value of 100 means 100 callbacks, each one of which represents 10ms worth
// of data, so the stats reporting frequency will be 1Hz (modulo failures).
constexpr static int kHistogramReportingInterval = 100;
mutable Mutex rtcp_counter_mutex_;
RtcpPacketTypeCounter rtcp_packet_type_counter_
RTC_GUARDED_BY(rtcp_counter_mutex_);
std::map<int, SdpAudioFormat> payload_type_map_;
};
void ChannelReceive::OnReceivedPayloadData(
rtc::ArrayView<const uint8_t> payload,
const RTPHeader& rtpHeader,
Timestamp receive_time) {
if (!playing_) {
// Avoid inserting into NetEQ when we are not playing. Count the
// packet as discarded.
// Tell source_tracker_ that the frame has been "delivered". Normally, this
// happens in AudioReceiveStreamInterface when audio frames are pulled out,
// but when playout is muted, nothing is pulling frames. The downside of
// this approach is that frames delivered this way won't be delayed for
// playout, and therefore will be unsynchronized with (a) audio delay when
// playing and (b) any audio/video synchronization. But the alternative is
// that muting playout also stops the SourceTracker from updating RtpSource
// information.
RtpPacketInfos::vector_type packet_vector = {
RtpPacketInfo(rtpHeader, receive_time)};
source_tracker_.OnFrameDelivered(RtpPacketInfos(packet_vector),
env_.clock().CurrentTime());
return;
}
// Push the incoming payload (parsed and ready for decoding) into NetEq.
if (payload.empty()) {
neteq_->InsertEmptyPacket(rtpHeader);
} else if (neteq_->InsertPacket(rtpHeader, payload, receive_time) < 0) {
RTC_DLOG(LS_ERROR) << "ChannelReceive::OnReceivedPayloadData() unable to "
"insert packet into NetEq; PT = "
<< static_cast<int>(rtpHeader.payloadType);
return;
}
TimeDelta round_trip_time = rtp_rtcp_->LastRtt().value_or(TimeDelta::Zero());
std::vector<uint16_t> nack_list = neteq_->GetNackList(round_trip_time.ms());
if (!nack_list.empty()) {
// Can't use nack_list.data() since it's not supported by all
// compilers.
ResendPackets(&(nack_list[0]), static_cast<int>(nack_list.size()));
}
}
void ChannelReceive::InitFrameTransformerDelegate(
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
RTC_DCHECK(frame_transformer);
RTC_DCHECK(!frame_transformer_delegate_);
RTC_DCHECK(worker_thread_->IsCurrent());
// Pass a callback to ChannelReceive::OnReceivedPayloadData, to be called by
// the delegate to receive transformed audio.
ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback
receive_audio_callback = [this](rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header,
Timestamp receive_time) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
OnReceivedPayloadData(packet, header, receive_time);
};
frame_transformer_delegate_ =
rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>(
std::move(receive_audio_callback), std::move(frame_transformer),
worker_thread_);
frame_transformer_delegate_->Init();
}
AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo(
int sample_rate_hz,
AudioFrame* audio_frame) {
TRACE_EVENT_BEGIN1("webrtc", "ChannelReceive::GetAudioFrameWithInfo",
"sample_rate_hz", sample_rate_hz);
RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_);
audio_frame->sample_rate_hz_ = sample_rate_hz;
env_.event_log().Log(std::make_unique<RtcEventAudioPlayout>(remote_ssrc_));
if ((neteq_->GetAudio(audio_frame) != NetEq::kOK) ||
!resampler_helper_.MaybeResample(sample_rate_hz, audio_frame)) {
RTC_DLOG(LS_ERROR)
<< "ChannelReceive::GetAudioFrame() PlayoutData10Ms() failed!";
// In all likelihood, the audio in this frame is garbage. We return an
// error so that the audio mixer module doesn't add it to the mix. As
// a result, it won't be played out and the actions skipped here are
// irrelevant.
TRACE_EVENT_END1("webrtc", "ChannelReceive::GetAudioFrameWithInfo", "error",
1);
return AudioMixer::Source::AudioFrameInfo::kError;
}
{
MutexLock lock(&call_stats_mutex_);
call_stats_.DecodedByNetEq(audio_frame->speech_type_, audio_frame->muted());
}
{
// Pass the audio buffers to an optional sink callback, before applying
// scaling/panning, as that applies to the mix operation.
// External recipients of the audio (e.g. via AudioTrack), will do their
// own mixing/dynamic processing.
MutexLock lock(&callback_mutex_);
if (audio_sink_) {
AudioSinkInterface::Data data(
audio_frame->data(), audio_frame->samples_per_channel_,
audio_frame->sample_rate_hz_, audio_frame->num_channels_,
audio_frame->timestamp_);
audio_sink_->OnData(data);
}
}
float output_gain = 1.0f;
{
MutexLock lock(&volume_settings_mutex_);
output_gain = _outputGain;
}
// Output volume scaling
if (output_gain < 0.99f || output_gain > 1.01f) {
// TODO(solenberg): Combine with mute state - this can cause clicks!
AudioFrameOperations::ScaleWithSat(output_gain, audio_frame);
}
// Measure audio level (0-9)
// TODO(henrik.lundin) Use the `muted` information here too.
// TODO(deadbeef): Use RmsLevel for `_outputAudioLevel` (see
// https://crbug.com/webrtc/7517).
_outputAudioLevel.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds);
if (capture_start_rtp_time_stamp_ < 0 && audio_frame->timestamp_ != 0) {
// The first frame with a valid rtp timestamp.
capture_start_rtp_time_stamp_ = audio_frame->timestamp_;
}
if (capture_start_rtp_time_stamp_ >= 0) {
// audio_frame.timestamp_ should be valid from now on.
// Compute elapsed time.
int64_t unwrap_timestamp =
rtp_ts_wraparound_handler_.Unwrap(audio_frame->timestamp_);
audio_frame->elapsed_time_ms_ =
(unwrap_timestamp - capture_start_rtp_time_stamp_) /
(GetRtpTimestampRateHz() / 1000);
{
MutexLock lock(&ts_stats_lock_);
// Compute ntp time.
audio_frame->ntp_time_ms_ =
ntp_estimator_.Estimate(audio_frame->timestamp_);
// `ntp_time_ms_` won't be valid until at least 2 RTCP SRs are received.
if (audio_frame->ntp_time_ms_ > 0) {
// Compute `capture_start_ntp_time_ms_` so that
// `capture_start_ntp_time_ms_` + `elapsed_time_ms_` == `ntp_time_ms_`
capture_start_ntp_time_ms_ =
audio_frame->ntp_time_ms_ - audio_frame->elapsed_time_ms_;
}
}
}
// Fill in local capture clock offset in `audio_frame->packet_infos_`.
RtpPacketInfos::vector_type packet_infos;
for (auto& packet_info : audio_frame->packet_infos_) {
RtpPacketInfo new_packet_info(packet_info);
if (packet_info.absolute_capture_time().has_value()) {
MutexLock lock(&ts_stats_lock_);
new_packet_info.set_local_capture_clock_offset(
capture_clock_offset_updater_.ConvertsToTimeDela(
capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset(
packet_info.absolute_capture_time()
->estimated_capture_clock_offset)));
}
packet_infos.push_back(std::move(new_packet_info));
}
audio_frame->packet_infos_ = RtpPacketInfos(std::move(packet_infos));
if (!audio_frame->packet_infos_.empty()) {
RtpPacketInfos infos_copy = audio_frame->packet_infos_;
Timestamp delivery_time = env_.clock().CurrentTime();
worker_thread_->PostTask(
SafeTask(worker_safety_.flag(), [this, infos_copy, delivery_time]() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
source_tracker_.OnFrameDelivered(infos_copy, delivery_time);
}));
}
++audio_frame_interval_count_;
if (audio_frame_interval_count_ >= kHistogramReportingInterval) {
audio_frame_interval_count_ = 0;
worker_thread_->PostTask(SafeTask(worker_safety_.flag(), [this]() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.TargetJitterBufferDelayMs",
neteq_->TargetDelayMs());
const int jitter_buffer_delay = neteq_->FilteredCurrentDelayMs();
RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDelayEstimateMs",
jitter_buffer_delay + playout_delay_ms_);
RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverJitterBufferDelayMs",
jitter_buffer_delay);
RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDeviceDelayMs",
playout_delay_ms_);
}));
}
TRACE_EVENT_END2("webrtc", "ChannelReceive::GetAudioFrameWithInfo", "gain",
output_gain, "muted", audio_frame->muted());
return audio_frame->muted() ? AudioMixer::Source::AudioFrameInfo::kMuted
: AudioMixer::Source::AudioFrameInfo::kNormal;
}
int ChannelReceive::PreferredSampleRate() const {
RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_);
const std::optional<NetEq::DecoderFormat> decoder =
neteq_->GetCurrentDecoderFormat();
const int last_packet_sample_rate_hz = decoder ? decoder->sample_rate_hz : 0;
// Return the bigger of playout and receive frequency in the ACM.
return std::max(last_packet_sample_rate_hz,
neteq_->last_output_sample_rate_hz());
}
ChannelReceive::ChannelReceive(
const Environment& env,
NetEqFactory* neteq_factory,
AudioDeviceModule* audio_device_module,
Transport* rtcp_send_transport,
uint32_t local_ssrc,
uint32_t remote_ssrc,
size_t jitter_buffer_max_packets,
bool jitter_buffer_fast_playout,
int jitter_buffer_min_delay_ms,
bool enable_non_sender_rtt,
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
std::optional<AudioCodecPairId> codec_pair_id,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
const webrtc::CryptoOptions& crypto_options,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
: env_(env),
worker_thread_(TaskQueueBase::Current()),
rtp_receive_statistics_(ReceiveStatistics::Create(&env_.clock())),
remote_ssrc_(remote_ssrc),
source_tracker_(&env_.clock()),
neteq_(CreateNetEq(neteq_factory,
codec_pair_id,
jitter_buffer_max_packets,
jitter_buffer_fast_playout,
jitter_buffer_min_delay_ms,
env_,
decoder_factory)),
_outputAudioLevel(),
ntp_estimator_(&env_.clock()),
playout_timestamp_rtp_(0),
playout_delay_ms_(0),
capture_start_rtp_time_stamp_(-1),
capture_start_ntp_time_ms_(-1),
_audioDeviceModulePtr(audio_device_module),
_outputGain(1.0f),
associated_send_channel_(nullptr),
frame_decryptor_(frame_decryptor),
crypto_options_(crypto_options),
absolute_capture_time_interpolator_(&env_.clock()) {
RTC_DCHECK(audio_device_module);
network_thread_checker_.Detach();
rtp_receive_statistics_->EnableRetransmitDetection(remote_ssrc_, true);
RtpRtcpInterface::Configuration configuration;
configuration.audio = true;
configuration.receiver_only = true;
configuration.outgoing_transport = rtcp_send_transport;
configuration.receive_statistics = rtp_receive_statistics_.get();
configuration.local_media_ssrc = local_ssrc;
configuration.rtcp_packet_type_counter_observer = this;
configuration.non_sender_rtt_measurement = enable_non_sender_rtt;
if (frame_transformer)
InitFrameTransformerDelegate(std::move(frame_transformer));
rtp_rtcp_ = std::make_unique<ModuleRtpRtcpImpl2>(env_, configuration);
rtp_rtcp_->SetRemoteSSRC(remote_ssrc_);
// Ensure that RTCP is enabled for the created channel.
rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound);
}
ChannelReceive::~ChannelReceive() {
RTC_DCHECK_RUN_ON(&construction_thread_);
// Resets the delegate's callback to ChannelReceive::OnReceivedPayloadData.
if (frame_transformer_delegate_)
frame_transformer_delegate_->Reset();
StopPlayout();
}
void ChannelReceive::SetSink(AudioSinkInterface* sink) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
MutexLock lock(&callback_mutex_);
audio_sink_ = sink;
}
void ChannelReceive::StartPlayout() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
playing_ = true;
}
void ChannelReceive::StopPlayout() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
playing_ = false;
_outputAudioLevel.ResetLevelFullRange();
neteq_->FlushBuffers();
}
std::optional<std::pair<int, SdpAudioFormat>> ChannelReceive::GetReceiveCodec()
const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
std::optional<NetEq::DecoderFormat> decoder =
neteq_->GetCurrentDecoderFormat();
if (!decoder) {
return std::nullopt;
}
return std::make_pair(decoder->payload_type, decoder->sdp_format);
}
void ChannelReceive::SetReceiveCodecs(
const std::map<int, SdpAudioFormat>& codecs) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
for (const auto& kv : codecs) {
RTC_DCHECK_GE(kv.second.clockrate_hz, 1000);
payload_type_frequencies_[kv.first] = kv.second.clockrate_hz;
}
payload_type_map_ = codecs;
neteq_->SetCodecs(codecs);
}
void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
// TODO(bugs.webrtc.org/11993): Expect to be called exclusively on the
// network thread. Once that's done, the same applies to
// UpdatePlayoutTimestamp and
int64_t now_ms = rtc::TimeMillis();
last_received_rtp_timestamp_ = packet.Timestamp();
last_received_rtp_system_time_ms_ = now_ms;
// Store playout timestamp for the received RTP packet
UpdatePlayoutTimestamp(false, now_ms);
const auto& it = payload_type_frequencies_.find(packet.PayloadType());
if (it == payload_type_frequencies_.end())
return;
// TODO(bugs.webrtc.org/7135): Set payload_type_frequency earlier, when packet
// is parsed.
RtpPacketReceived packet_copy(packet);
packet_copy.set_payload_type_frequency(it->second);
rtp_receive_statistics_->OnRtpPacket(packet_copy);
RTPHeader header;
packet_copy.GetHeader(&header);
// Interpolates absolute capture timestamp RTP header extension.
header.extension.absolute_capture_time =
absolute_capture_time_interpolator_.OnReceivePacket(
AbsoluteCaptureTimeInterpolator::GetSource(header.ssrc,
header.arrOfCSRCs),
header.timestamp,
rtc::saturated_cast<uint32_t>(packet_copy.payload_type_frequency()),
header.extension.absolute_capture_time);
ReceivePacket(packet_copy.data(), packet_copy.size(), header,
packet.arrival_time());
}
void ChannelReceive::ReceivePacket(const uint8_t* packet,
size_t packet_length,
const RTPHeader& header,
Timestamp receive_time) {
const uint8_t* payload = packet + header.headerLength;
RTC_DCHECK_GE(packet_length, header.headerLength);
size_t payload_length = packet_length - header.headerLength;
size_t payload_data_length = payload_length - header.paddingLength;
// E2EE Custom Audio Frame Decryption (This is optional).
// Keep this buffer around for the lifetime of the OnReceivedPayloadData call.
rtc::Buffer decrypted_audio_payload;
if (frame_decryptor_ != nullptr) {
const size_t max_plaintext_size = frame_decryptor_->GetMaxPlaintextByteSize(
cricket::MEDIA_TYPE_AUDIO, payload_length);
decrypted_audio_payload.SetSize(max_plaintext_size);
const std::vector<uint32_t> csrcs(header.arrOfCSRCs,
header.arrOfCSRCs + header.numCSRCs);
const FrameDecryptorInterface::Result decrypt_result =
frame_decryptor_->Decrypt(
cricket::MEDIA_TYPE_AUDIO, csrcs,
/*additional_data=*/
nullptr,
rtc::ArrayView<const uint8_t>(payload, payload_data_length),
decrypted_audio_payload);
if (decrypt_result.IsOk()) {
decrypted_audio_payload.SetSize(decrypt_result.bytes_written);
} else {
// Interpret failures as a silent frame.
decrypted_audio_payload.SetSize(0);
}
payload = decrypted_audio_payload.data();
payload_data_length = decrypted_audio_payload.size();
} else if (crypto_options_.sframe.require_frame_encryption) {
RTC_DLOG(LS_ERROR)
<< "FrameDecryptor required but not set, dropping packet";
payload_data_length = 0;
}
rtc::ArrayView<const uint8_t> payload_data(payload, payload_data_length);
if (frame_transformer_delegate_) {
// Asynchronously transform the received payload. After the payload is
// transformed, the delegate will call OnReceivedPayloadData to handle it.
char buf[1024];
rtc::SimpleStringBuilder mime_type(buf);
auto it = payload_type_map_.find(header.payloadType);
mime_type << MediaTypeToString(cricket::MEDIA_TYPE_AUDIO) << "/"
<< (it != payload_type_map_.end() ? it->second.name
: "x-unknown");
frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_,
mime_type.str(), receive_time);
} else {
OnReceivedPayloadData(payload_data, header, receive_time);
}
}
void ChannelReceive::ReceivedRTCPPacket(const uint8_t* data, size_t length) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
// TODO(bugs.webrtc.org/11993): Expect to be called exclusively on the
// network thread.
// Store playout timestamp for the received RTCP packet
UpdatePlayoutTimestamp(true, rtc::TimeMillis());
// Deliver RTCP packet to RTP/RTCP module for parsing
rtp_rtcp_->IncomingRtcpPacket(rtc::MakeArrayView(data, length));
std::optional<TimeDelta> rtt = rtp_rtcp_->LastRtt();
if (!rtt.has_value()) {
// Waiting for valid RTT.
return;
}
std::optional<RtpRtcpInterface::SenderReportStats> last_sr =
rtp_rtcp_->GetSenderReportStats();
if (!last_sr.has_value()) {
// Waiting for RTCP.
return;
}
{
MutexLock lock(&ts_stats_lock_);
ntp_estimator_.UpdateRtcpTimestamp(*rtt, last_sr->last_remote_ntp_timestamp,
last_sr->last_remote_rtp_timestamp);
std::optional<int64_t> remote_to_local_clock_offset =
ntp_estimator_.EstimateRemoteToLocalClockOffset();
if (remote_to_local_clock_offset.has_value()) {
capture_clock_offset_updater_.SetRemoteToLocalClockOffset(
*remote_to_local_clock_offset);
}
}
}
int ChannelReceive::GetSpeechOutputLevelFullRange() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return _outputAudioLevel.LevelFullRange();
}
double ChannelReceive::GetTotalOutputEnergy() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return _outputAudioLevel.TotalEnergy();
}
double ChannelReceive::GetTotalOutputDuration() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return _outputAudioLevel.TotalDuration();
}
void ChannelReceive::SetChannelOutputVolumeScaling(float scaling) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
MutexLock lock(&volume_settings_mutex_);
_outputGain = scaling;
}
void ChannelReceive::RegisterReceiverCongestionControlObjects(
PacketRouter* packet_router) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
RTC_DCHECK(packet_router);
RTC_DCHECK(!packet_router_);
constexpr bool remb_candidate = false;
packet_router->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
packet_router_ = packet_router;
}
void ChannelReceive::ResetReceiverCongestionControlObjects() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
RTC_DCHECK(packet_router_);
packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get());
packet_router_ = nullptr;
}
CallReceiveStatistics ChannelReceive::GetRTCPStatistics() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
CallReceiveStatistics stats;
// The jitter statistics is updated for each received RTP packet and is based
// on received packets.
RtpReceiveStats rtp_stats;
StreamStatistician* statistician =
rtp_receive_statistics_->GetStatistician(remote_ssrc_);
if (statistician) {
rtp_stats = statistician->GetStats();
}
stats.cumulativeLost = rtp_stats.packets_lost;
stats.jitterSamples = rtp_stats.jitter;
// Data counters.
if (statistician) {
stats.payload_bytes_received = rtp_stats.packet_counter.payload_bytes;
stats.header_and_padding_bytes_received =
rtp_stats.packet_counter.header_bytes +
rtp_stats.packet_counter.padding_bytes;
stats.packetsReceived = rtp_stats.packet_counter.packets;
stats.last_packet_received = rtp_stats.last_packet_received;
} else {
stats.payload_bytes_received = 0;
stats.header_and_padding_bytes_received = 0;
stats.packetsReceived = 0;
stats.last_packet_received = std::nullopt;
}
{
MutexLock lock(&rtcp_counter_mutex_);
stats.nacks_sent = rtcp_packet_type_counter_.nack_packets;
}
// Timestamps.
{
MutexLock lock(&ts_stats_lock_);
stats.capture_start_ntp_time_ms_ = capture_start_ntp_time_ms_;
}
std::optional<RtpRtcpInterface::SenderReportStats> rtcp_sr_stats =
rtp_rtcp_->GetSenderReportStats();
if (rtcp_sr_stats.has_value()) {
stats.last_sender_report_timestamp = rtcp_sr_stats->last_arrival_timestamp;
stats.last_sender_report_utc_timestamp =
Clock::NtpToUtc(rtcp_sr_stats->last_arrival_ntp_timestamp);
stats.last_sender_report_remote_utc_timestamp =
Clock::NtpToUtc(rtcp_sr_stats->last_remote_ntp_timestamp);
stats.sender_reports_packets_sent = rtcp_sr_stats->packets_sent;
stats.sender_reports_bytes_sent = rtcp_sr_stats->bytes_sent;
stats.sender_reports_reports_count = rtcp_sr_stats->reports_count;
}
std::optional<RtpRtcpInterface::NonSenderRttStats> non_sender_rtt_stats =
rtp_rtcp_->GetNonSenderRttStats();
if (non_sender_rtt_stats.has_value()) {
stats.round_trip_time = non_sender_rtt_stats->round_trip_time;
stats.round_trip_time_measurements =
non_sender_rtt_stats->round_trip_time_measurements;
stats.total_round_trip_time = non_sender_rtt_stats->total_round_trip_time;
}
return stats;
}
void ChannelReceive::SetNACKStatus(bool enable, int max_packets) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
// None of these functions can fail.
if (enable) {
rtp_receive_statistics_->SetMaxReorderingThreshold(remote_ssrc_,
max_packets);
neteq_->EnableNack(max_packets);
} else {
rtp_receive_statistics_->SetMaxReorderingThreshold(
remote_ssrc_, kDefaultMaxReorderingThreshold);
neteq_->DisableNack();
}
}
void ChannelReceive::SetRtcpMode(webrtc::RtcpMode mode) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
rtp_rtcp_->SetRTCPStatus(mode);
}
void ChannelReceive::SetNonSenderRttMeasurement(bool enabled) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
rtp_rtcp_->SetNonSenderRttMeasurement(enabled);
}
// Called when we are missing one or more packets.
int ChannelReceive::ResendPackets(const uint16_t* sequence_numbers,
int length) {
return rtp_rtcp_->SendNACK(sequence_numbers, length);
}
void ChannelReceive::RtcpPacketTypesCounterUpdated(
uint32_t ssrc,
const RtcpPacketTypeCounter& packet_counter) {
if (ssrc != remote_ssrc_) {
return;
}
MutexLock lock(&rtcp_counter_mutex_);
rtcp_packet_type_counter_ = packet_counter;
}
void ChannelReceive::SetAssociatedSendChannel(
const ChannelSendInterface* channel) {
RTC_DCHECK_RUN_ON(&network_thread_checker_);
associated_send_channel_ = channel;
}
void ChannelReceive::SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (!frame_transformer) {
RTC_DCHECK_NOTREACHED() << "Not setting the transformer?";
return;
}
if (frame_transformer_delegate_) {
// Depending on when the channel is created, the transformer might be set
// twice. Don't replace the delegate if it was already initialized.
// TODO(crbug.com/webrtc/15674): Prevent multiple calls during
// reconfiguration.
RTC_CHECK_EQ(frame_transformer_delegate_->FrameTransformer(),
frame_transformer);
return;
}
InitFrameTransformerDelegate(std::move(frame_transformer));
}
void ChannelReceive::SetFrameDecryptor(
rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
// TODO(bugs.webrtc.org/11993): Expect to be called on the network thread.
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
frame_decryptor_ = std::move(frame_decryptor);
}
void ChannelReceive::OnLocalSsrcChange(uint32_t local_ssrc) {
// TODO(bugs.webrtc.org/11993): Expect to be called on the network thread.
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
rtp_rtcp_->SetLocalSsrc(local_ssrc);
}
uint32_t ChannelReceive::GetLocalSsrc() const {
// TODO(bugs.webrtc.org/11993): Expect to be called on the network thread.
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return rtp_rtcp_->local_media_ssrc();
}
NetworkStatistics ChannelReceive::GetNetworkStatistics(
bool get_and_clear_legacy_stats) const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
NetworkStatistics acm_stat;
NetEqNetworkStatistics neteq_stat;
if (get_and_clear_legacy_stats) {
// NetEq function always returns zero, so we don't check the return value.
neteq_->NetworkStatistics(&neteq_stat);
acm_stat.currentExpandRate = neteq_stat.expand_rate;
acm_stat.currentSpeechExpandRate = neteq_stat.speech_expand_rate;
acm_stat.currentPreemptiveRate = neteq_stat.preemptive_rate;
acm_stat.currentAccelerateRate = neteq_stat.accelerate_rate;
acm_stat.currentSecondaryDecodedRate = neteq_stat.secondary_decoded_rate;
acm_stat.currentSecondaryDiscardedRate =
neteq_stat.secondary_discarded_rate;
acm_stat.meanWaitingTimeMs = neteq_stat.mean_waiting_time_ms;
acm_stat.maxWaitingTimeMs = neteq_stat.max_waiting_time_ms;
} else {
neteq_stat = neteq_->CurrentNetworkStatistics();
acm_stat.currentExpandRate = 0;
acm_stat.currentSpeechExpandRate = 0;
acm_stat.currentPreemptiveRate = 0;
acm_stat.currentAccelerateRate = 0;
acm_stat.currentSecondaryDecodedRate = 0;
acm_stat.currentSecondaryDiscardedRate = 0;
acm_stat.meanWaitingTimeMs = -1;
acm_stat.maxWaitingTimeMs = 1;
}
acm_stat.currentBufferSize = neteq_stat.current_buffer_size_ms;
acm_stat.preferredBufferSize = neteq_stat.preferred_buffer_size_ms;
acm_stat.jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false;
NetEqLifetimeStatistics neteq_lifetime_stat = neteq_->GetLifetimeStatistics();
acm_stat.totalSamplesReceived = neteq_lifetime_stat.total_samples_received;
acm_stat.concealedSamples = neteq_lifetime_stat.concealed_samples;
acm_stat.silentConcealedSamples =
neteq_lifetime_stat.silent_concealed_samples;
acm_stat.concealmentEvents = neteq_lifetime_stat.concealment_events;
acm_stat.jitterBufferDelayMs = neteq_lifetime_stat.jitter_buffer_delay_ms;
acm_stat.jitterBufferTargetDelayMs =
neteq_lifetime_stat.jitter_buffer_target_delay_ms;
acm_stat.jitterBufferMinimumDelayMs =
neteq_lifetime_stat.jitter_buffer_minimum_delay_ms;
acm_stat.jitterBufferEmittedCount =
neteq_lifetime_stat.jitter_buffer_emitted_count;
acm_stat.delayedPacketOutageSamples =
neteq_lifetime_stat.delayed_packet_outage_samples;
acm_stat.relativePacketArrivalDelayMs =
neteq_lifetime_stat.relative_packet_arrival_delay_ms;
acm_stat.interruptionCount = neteq_lifetime_stat.interruption_count;
acm_stat.totalInterruptionDurationMs =
neteq_lifetime_stat.total_interruption_duration_ms;
acm_stat.insertedSamplesForDeceleration =
neteq_lifetime_stat.inserted_samples_for_deceleration;
acm_stat.removedSamplesForAcceleration =
neteq_lifetime_stat.removed_samples_for_acceleration;
acm_stat.fecPacketsReceived = neteq_lifetime_stat.fec_packets_received;
acm_stat.fecPacketsDiscarded = neteq_lifetime_stat.fec_packets_discarded;
acm_stat.totalProcessingDelayUs =
neteq_lifetime_stat.total_processing_delay_us;
acm_stat.packetsDiscarded = neteq_lifetime_stat.packets_discarded;
NetEqOperationsAndState neteq_operations_and_state =
neteq_->GetOperationsAndState();
acm_stat.packetBufferFlushes =
neteq_operations_and_state.packet_buffer_flushes;
return acm_stat;
}
AudioDecodingCallStats ChannelReceive::GetDecodingCallStatistics() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
MutexLock lock(&call_stats_mutex_);
return call_stats_.GetDecodingStatistics();
}
uint32_t ChannelReceive::GetDelayEstimate() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
// Return the current jitter buffer delay + playout delay.
return neteq_->FilteredCurrentDelayMs() + playout_delay_ms_;
}
bool ChannelReceive::SetMinimumPlayoutDelay(int delay_ms) {
// TODO(bugs.webrtc.org/11993): This should run on the network thread.
// We get here via RtpStreamsSynchronizer. Once that's done, many (all?) of
// these locks aren't needed.
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
// Limit to range accepted by both VoE and ACM, so we're at least getting as
// close as possible, instead of failing.
delay_ms = rtc::SafeClamp(delay_ms, kVoiceEngineMinMinPlayoutDelayMs,
kVoiceEngineMaxMinPlayoutDelayMs);
if (!neteq_->SetMinimumDelay(delay_ms)) {
RTC_DLOG(LS_ERROR)
<< "SetMinimumPlayoutDelay() failed to set min playout delay "
<< delay_ms;
return false;
}
return true;
}
bool ChannelReceive::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp,
int64_t* time_ms) const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (!playout_timestamp_rtp_time_ms_)
return false;
*rtp_timestamp = playout_timestamp_rtp_;
*time_ms = playout_timestamp_rtp_time_ms_.value();
return true;
}
void ChannelReceive::SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms,
int64_t time_ms) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
playout_timestamp_ntp_ = ntp_timestamp_ms;
playout_timestamp_ntp_time_ms_ = time_ms;
}
std::optional<int64_t> ChannelReceive::GetCurrentEstimatedPlayoutNtpTimestampMs(
int64_t now_ms) const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (!playout_timestamp_ntp_ || !playout_timestamp_ntp_time_ms_)
return std::nullopt;
int64_t elapsed_ms = now_ms - *playout_timestamp_ntp_time_ms_;
return *playout_timestamp_ntp_ + elapsed_ms;
}
bool ChannelReceive::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
env_.event_log().Log(
std::make_unique<RtcEventNetEqSetMinimumDelay>(remote_ssrc_, delay_ms));
return neteq_->SetBaseMinimumDelayMs(delay_ms);
}
int ChannelReceive::GetBaseMinimumPlayoutDelayMs() const {
return neteq_->GetBaseMinimumDelayMs();
}
std::optional<Syncable::Info> ChannelReceive::GetSyncInfo() const {
// TODO(bugs.webrtc.org/11993): This should run on the network thread.
// We get here via RtpStreamsSynchronizer. Once that's done, many of
// these locks aren't needed.
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Syncable::Info info;
std::optional<RtpRtcpInterface::SenderReportStats> last_sr =
rtp_rtcp_->GetSenderReportStats();
if (!last_sr.has_value()) {
return std::nullopt;
}
info.capture_time_ntp_secs = last_sr->last_remote_ntp_timestamp.seconds();
info.capture_time_ntp_frac = last_sr->last_remote_ntp_timestamp.fractions();
info.capture_time_source_clock = last_sr->last_remote_rtp_timestamp;
if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_ms_) {
return std::nullopt;
}
info.latest_received_capture_timestamp = *last_received_rtp_timestamp_;
info.latest_receive_time_ms = *last_received_rtp_system_time_ms_;
int jitter_buffer_delay = neteq_->FilteredCurrentDelayMs();
info.current_delay_ms = jitter_buffer_delay + playout_delay_ms_;
return info;
}
void ChannelReceive::UpdatePlayoutTimestamp(bool rtcp, int64_t now_ms) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
// TODO(bugs.webrtc.org/11993): Expect to be called exclusively on the
// network thread. Once that's done, we won't need video_sync_lock_.
jitter_buffer_playout_timestamp_ = neteq_->GetPlayoutTimestamp();
if (!jitter_buffer_playout_timestamp_) {
// This can happen if this channel has not received any RTP packets. In
// this case, NetEq is not capable of computing a playout timestamp.
return;
}
uint16_t delay_ms = 0;
if (_audioDeviceModulePtr->PlayoutDelay(&delay_ms) == -1) {
RTC_DLOG(LS_WARNING)
<< "ChannelReceive::UpdatePlayoutTimestamp() failed to read"
" playout delay from the ADM";
return;
}
RTC_DCHECK(jitter_buffer_playout_timestamp_);
uint32_t playout_timestamp = *jitter_buffer_playout_timestamp_;
// Remove the playout delay.
playout_timestamp -= (delay_ms * (GetRtpTimestampRateHz() / 1000));
if (!rtcp && playout_timestamp != playout_timestamp_rtp_) {
playout_timestamp_rtp_ = playout_timestamp;
playout_timestamp_rtp_time_ms_ = now_ms;
}
playout_delay_ms_ = delay_ms;
}
int ChannelReceive::GetRtpTimestampRateHz() const {
const auto decoder_format = neteq_->GetCurrentDecoderFormat();
// Default to the playout frequency if we've not gotten any packets yet.
// TODO(ossu): Zero clockrate can only happen if we've added an external
// decoder for a format we don't support internally. Remove once that way of
// adding decoders is gone!
// TODO(kwiberg): `decoder_format->sdp_format.clockrate_hz` is an RTP
// clockrate as it should, but `neteq_->last_output_sample_rate_hz()` is a
// codec sample rate, which is not always the same thing.
return (decoder_format && decoder_format->sdp_format.clockrate_hz != 0)
? decoder_format->sdp_format.clockrate_hz
: neteq_->last_output_sample_rate_hz();
}
std::vector<RtpSource> ChannelReceive::GetSources() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return source_tracker_.GetSources();
}
} // namespace
std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive(
const Environment& env,
NetEqFactory* neteq_factory,
AudioDeviceModule* audio_device_module,
Transport* rtcp_send_transport,
uint32_t local_ssrc,
uint32_t remote_ssrc,
size_t jitter_buffer_max_packets,
bool jitter_buffer_fast_playout,
int jitter_buffer_min_delay_ms,
bool enable_non_sender_rtt,
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
std::optional<AudioCodecPairId> codec_pair_id,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
const webrtc::CryptoOptions& crypto_options,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
return std::make_unique<ChannelReceive>(
env, neteq_factory, audio_device_module, rtcp_send_transport, local_ssrc,
remote_ssrc, jitter_buffer_max_packets, jitter_buffer_fast_playout,
jitter_buffer_min_delay_ms, enable_non_sender_rtt, decoder_factory,
codec_pair_id, std::move(frame_decryptor), crypto_options,
std::move(frame_transformer));
}
} // namespace voe
} // namespace webrtc