blob: 314afd7dcd6bb79f0c4e185f16fd3c98cd6734b3 [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:261/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:2612
Yves Gerey988cc082018-10-23 10:03:0113#include <assert.h>
Jonas Olssona4d87372019-07-05 17:08:3314
Jonathan Yu36344a02017-07-30 08:55:3415#include <algorithm>
Yves Gerey988cc082018-10-23 10:03:0116#include <cstdint>
Jonathan Yu36344a02017-07-30 08:55:3417
Niels Möller2edab4c2018-10-22 07:48:0818#include "absl/strings/match.h"
Yves Gerey988cc082018-10-23 10:03:0119#include "api/array_view.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3120#include "modules/audio_coding/acm2/acm_receiver.h"
21#include "modules/audio_coding/acm2/acm_resampler.h"
Fredrik Solenbergbbf21a32018-04-12 20:44:0922#include "modules/include/module_common_types.h"
Yves Gerey988cc082018-10-23 10:03:0123#include "modules/include/module_common_types_public.h"
24#include "rtc_base/buffer.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3125#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 17:11:0026#include "rtc_base/critical_section.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3127#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 09:42:2628#include "rtc_base/numerics/safe_conversions.h"
Yves Gerey988cc082018-10-23 10:03:0129#include "rtc_base/thread_annotations.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3130#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:2631
32namespace webrtc {
33
kwibergc13ded52016-06-17 13:00:4534namespace {
35
Per Åhgren4f2e9402019-10-04 09:06:1536// Initial size for the buffer in InputBuffer. This matches 6 channels of 10 ms
37// 48 kHz data.
38constexpr size_t kInitialInputDataBufferSize = 6 * 480;
39
kwibergc13ded52016-06-17 13:00:4540class AudioCodingModuleImpl final : public AudioCodingModule {
41 public:
42 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
43 ~AudioCodingModuleImpl() override;
44
45 /////////////////////////////////////////
46 // Sender
47 //
48
kwiberg24c7c122016-09-28 18:57:1049 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
50 modifier) override;
kwibergc13ded52016-06-17 13:00:4551
kwibergc13ded52016-06-17 13:00:4552 // Register a transport callback which will be
53 // called to deliver the encoded buffers.
54 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
55
56 // Add 10 ms of raw (PCM) audio data to the encoder.
57 int Add10MsData(const AudioFrame& audio_frame) override;
58
59 /////////////////////////////////////////
kwibergc13ded52016-06-17 13:00:4560 // (FEC) Forward Error Correction (codec internal)
61 //
62
kwibergc13ded52016-06-17 13:00:4563 // Set target packet loss rate
64 int SetPacketLossRate(int loss_rate) override;
65
66 /////////////////////////////////////////
67 // (VAD) Voice Activity Detection
68 // and
69 // (CNG) Comfort Noise Generation
70 //
71
kwibergc13ded52016-06-17 13:00:4572 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
73
74 /////////////////////////////////////////
75 // Receiver
76 //
77
78 // Initialize receiver, resets codec database etc.
79 int InitializeReceiver() override;
80
kwiberg1c07c702017-03-27 14:15:4981 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
82
kwibergc13ded52016-06-17 13:00:4583 // Incoming packet from network parsed and ready for decode.
84 int IncomingPacket(const uint8_t* incoming_payload,
85 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 14:21:4786 const RTPHeader& rtp_info) override;
kwibergc13ded52016-06-17 13:00:4587
kwibergc13ded52016-06-17 13:00:4588 // Get 10 milliseconds of raw audio data to play out, and
89 // automatic resample to the requested frequency if > 0.
90 int PlayoutData10Ms(int desired_freq_hz,
91 AudioFrame* audio_frame,
92 bool* muted) override;
kwibergc13ded52016-06-17 13:00:4593
94 /////////////////////////////////////////
95 // Statistics
96 //
97
98 int GetNetworkStatistics(NetworkStatistics* statistics) override;
99
ivoce1198e02017-09-08 15:13:19100 ANAStats GetANAStats() const override;
101
kwibergc13ded52016-06-17 13:00:45102 private:
103 struct InputData {
Per Åhgren4f2e9402019-10-04 09:06:15104 InputData() : buffer(kInitialInputDataBufferSize) {}
kwibergc13ded52016-06-17 13:00:45105 uint32_t input_timestamp;
106 const int16_t* audio;
107 size_t length_per_channel;
108 size_t audio_channel;
109 // If a re-mix is required (up or down), this buffer will store a re-mixed
110 // version of the input.
Per Åhgren4f2e9402019-10-04 09:06:15111 std::vector<int16_t> buffer;
kwibergc13ded52016-06-17 13:00:45112 };
113
Per Åhgren4f2e9402019-10-04 09:06:15114 InputData input_data_ RTC_GUARDED_BY(acm_crit_sect_);
115
kwibergc13ded52016-06-17 13:00:45116 // This member class writes values to the named UMA histogram, but only if
117 // the value has changed since the last time (and always for the first call).
118 class ChangeLogger {
119 public:
120 explicit ChangeLogger(const std::string& histogram_name)
121 : histogram_name_(histogram_name) {}
122 // Logs the new value if it is different from the last logged value, or if
123 // this is the first call.
124 void MaybeLog(int value);
125
126 private:
127 int last_value_ = 0;
128 int first_time_ = true;
129 const std::string histogram_name_;
130 };
131
kwibergc13ded52016-06-17 13:00:45132 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 14:53:45133 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45134 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 14:53:45135 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45136
danilchap56359be2017-09-07 14:53:45137 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45138
139 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 14:53:45140 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45141
142 // Preprocessing of input audio, including resampling and down-mixing if
143 // required, before pushing audio into encoder's buffer.
144 //
145 // in_frame: input audio-frame
146 // ptr_out: pointer to output audio_frame. If no preprocessing is required
147 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
148 // |preprocess_frame_|.
149 //
150 // Return value:
151 // -1: if encountering an error.
152 // 0: otherwise.
153 int PreprocessToAddData(const AudioFrame& in_frame,
154 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 14:53:45155 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45156
157 // Change required states after starting to receive the codec corresponding
158 // to |index|.
159 int UpdateUponReceivingCodec(int index);
160
161 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 14:53:45162 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 14:53:45163 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
164 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
165 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45166 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 14:53:45167 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45168
Karl Wiberg49c33ce2018-11-12 13:21:58169 // Current encoder stack, provided by a call to RegisterEncoder.
danilchap56359be2017-09-07 14:53:45170 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45171
kwibergc13ded52016-06-17 13:00:45172 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 14:53:45173 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45174
danilchap56359be2017-09-07 14:53:45175 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45176
danilchap56359be2017-09-07 14:53:45177 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
178 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45179
danilchap56359be2017-09-07 14:53:45180 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
181 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
182 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45183
184 rtc::CriticalSection callback_crit_sect_;
185 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 14:53:45186 RTC_GUARDED_BY(callback_crit_sect_);
187 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 13:00:45188
189 int codec_histogram_bins_log_[static_cast<size_t>(
190 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
191 int number_of_consecutive_empty_packets_;
192};
193
194// Adds a codec usage sample to the histogram.
195void UpdateCodecTypeHistogram(size_t codec_type) {
196 RTC_HISTOGRAM_ENUMERATION(
197 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
198 static_cast<int>(
199 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
200}
201
kwibergc13ded52016-06-17 13:00:45202// Stereo-to-mono can be used as in-place.
Per Åhgren4f2e9402019-10-04 09:06:15203void DownMix(const AudioFrame& frame,
204 size_t length_out_buff,
205 int16_t* out_buff) {
yujo36b1a5f2017-06-12 19:45:32206 RTC_DCHECK_EQ(frame.num_channels_, 2);
207 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
208
209 if (!frame.muted()) {
210 const int16_t* frame_data = frame.data();
211 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
Yves Gerey665174f2018-06-19 13:03:05212 out_buff[n] =
213 static_cast<int16_t>((static_cast<int32_t>(frame_data[2 * n]) +
214 static_cast<int32_t>(frame_data[2 * n + 1])) >>
215 1);
yujo36b1a5f2017-06-12 19:45:32216 }
217 } else {
Jonathan Yu36344a02017-07-30 08:55:34218 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 13:00:45219 }
kwibergc13ded52016-06-17 13:00:45220}
221
Per Åhgren4f2e9402019-10-04 09:06:15222// Remixes the input frame to an output data vector. The output vector is
223// resized if needed.
224void ReMix(const AudioFrame& input,
225 size_t num_output_channels,
226 std::vector<int16_t>* output) {
227 const size_t output_size = num_output_channels * input.samples_per_channel_;
yujo36b1a5f2017-06-12 19:45:32228
Per Åhgren4f2e9402019-10-04 09:06:15229 if (output->size() != output_size) {
230 output->resize(output_size);
kwibergc13ded52016-06-17 13:00:45231 }
Per Åhgren4f2e9402019-10-04 09:06:15232
233 // For muted frames, fill the frame with zeros.
234 if (input.muted()) {
235 std::fill(output->begin(), output->end(), 0);
236 return;
237 }
238
239 // Ensure that the special case of zero input channels is handled correctly
240 // (zero samples per channel is already handled correctly in the code below).
241 if (input.num_channels_ == 0) {
242 return;
243 }
244
245 const int16_t* input_data = input.data();
246 size_t in_index = 0;
247 size_t out_index = 0;
248
249 // When upmixing is needed, duplicate the last channel of the input.
250 if (input.num_channels_ < num_output_channels) {
251 for (size_t k = 0; k < input.samples_per_channel_; ++k) {
252 for (size_t j = 0; j < input.num_channels_; ++j) {
253 (*output)[out_index++] = input_data[in_index++];
254 }
255 RTC_DCHECK_GT(in_index, 0);
256 const int16_t value_last_channel = input_data[in_index - 1];
257 for (size_t j = input.num_channels_; j < num_output_channels; ++j) {
258 (*output)[out_index++] = value_last_channel;
259 }
260 }
261 return;
262 }
263
264 // When downmixing is needed, and the input is stereo, average the channels.
265 if (input.num_channels_ == 2) {
266 for (size_t n = 0; n < input.samples_per_channel_; ++n) {
267 (*output)[n] =
268 static_cast<int16_t>((static_cast<int32_t>(input_data[2 * n]) +
269 static_cast<int32_t>(input_data[2 * n + 1])) >>
270 1);
271 }
272 return;
273 }
274
275 // When downmixing is needed, and the input is multichannel, drop the surplus
276 // channels.
277 const size_t num_channels_to_drop = input.num_channels_ - num_output_channels;
278 for (size_t k = 0; k < input.samples_per_channel_; ++k) {
279 for (size_t j = 0; j < num_output_channels; ++j) {
280 (*output)[out_index++] = input_data[in_index++];
281 }
282 in_index += num_channels_to_drop;
283 }
kwibergc13ded52016-06-17 13:00:45284}
285
kwibergc13ded52016-06-17 13:00:45286void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
287 if (value != last_value_ || first_time_) {
288 first_time_ = false;
289 last_value_ = value;
290 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
291 }
292}
293
294AudioCodingModuleImpl::AudioCodingModuleImpl(
295 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 14:37:11296 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 13:00:45297 expected_in_ts_(0xD87F3F9F),
298 receiver_(config),
299 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
kwibergc13ded52016-06-17 13:00:45300 encoder_stack_(nullptr),
301 previous_pltype_(255),
302 receiver_initialized_(false),
303 first_10ms_data_(false),
304 first_frame_(true),
305 packetization_callback_(NULL),
306 vad_callback_(NULL),
307 codec_histogram_bins_log_(),
308 number_of_consecutive_empty_packets_(0) {
309 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 10:09:25310 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 13:00:45311 }
Mirko Bonadei675513b2017-11-09 10:09:25312 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 13:00:45313}
314
315AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
316
317int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
318 AudioEncoder::EncodedInfo encoded_info;
319 uint8_t previous_pltype;
320
321 // Check if there is an encoder before.
322 if (!HaveValidEncoder("Process"))
323 return -1;
324
Yves Gerey665174f2018-06-19 13:03:05325 if (!first_frame_) {
deadbeeffcada902016-08-24 19:45:13326 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 16:34:22327 << "Time should not move backwards";
328 }
329
kwibergc13ded52016-06-17 13:00:45330 // Scale the timestamp to the codec's RTP timestamp rate.
331 uint32_t rtp_timestamp =
Karl Wiberg053c3712019-05-16 13:24:17332 first_frame_
333 ? input_data.input_timestamp
334 : last_rtp_timestamp_ +
335 rtc::dchecked_cast<uint32_t>(rtc::CheckedDivExact(
336 int64_t{input_data.input_timestamp - last_timestamp_} *
337 encoder_stack_->RtpTimestampRateHz(),
338 int64_t{encoder_stack_->SampleRateHz()}));
kwibergc13ded52016-06-17 13:00:45339 last_timestamp_ = input_data.input_timestamp;
340 last_rtp_timestamp_ = rtp_timestamp;
341 first_frame_ = false;
342
343 // Clear the buffer before reuse - encoded data will get appended.
344 encode_buffer_.Clear();
345 encoded_info = encoder_stack_->Encode(
Yves Gerey665174f2018-06-19 13:03:05346 rtp_timestamp,
347 rtc::ArrayView<const int16_t>(
348 input_data.audio,
349 input_data.audio_channel * input_data.length_per_channel),
kwibergc13ded52016-06-17 13:00:45350 &encode_buffer_);
351
352 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
353 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
354 // Not enough data.
355 return 0;
356 }
357 previous_pltype = previous_pltype_; // Read it while we have the critsect.
358
359 // Log codec type to histogram once every 500 packets.
360 if (encoded_info.encoded_bytes == 0) {
361 ++number_of_consecutive_empty_packets_;
362 } else {
363 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
364 codec_histogram_bins_log_[codec_type] +=
365 number_of_consecutive_empty_packets_ + 1;
366 number_of_consecutive_empty_packets_ = 0;
367 if (codec_histogram_bins_log_[codec_type] >= 500) {
368 codec_histogram_bins_log_[codec_type] -= 500;
369 UpdateCodecTypeHistogram(codec_type);
370 }
371 }
372
Niels Möller87e2d782019-03-07 09:18:23373 AudioFrameType frame_type;
kwibergc13ded52016-06-17 13:00:45374 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
Niels Möllerc936cb62019-03-19 13:10:16375 frame_type = AudioFrameType::kEmptyFrame;
kwibergc13ded52016-06-17 13:00:45376 encoded_info.payload_type = previous_pltype;
377 } else {
kwibergaf476c72016-11-28 23:21:39378 RTC_DCHECK_GT(encode_buffer_.size(), 0);
Niels Möllerc936cb62019-03-19 13:10:16379 frame_type = encoded_info.speech ? AudioFrameType::kAudioFrameSpeech
380 : AudioFrameType::kAudioFrameCN;
kwibergc13ded52016-06-17 13:00:45381 }
382
383 {
384 rtc::CritScope lock(&callback_crit_sect_);
385 if (packetization_callback_) {
386 packetization_callback_->SendData(
387 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
Niels Möllerc35b6e62019-04-25 14:31:18388 encode_buffer_.data(), encode_buffer_.size());
kwibergc13ded52016-06-17 13:00:45389 }
390
391 if (vad_callback_) {
392 // Callback with VAD decision.
393 vad_callback_->InFrameType(frame_type);
394 }
395 }
396 previous_pltype_ = encoded_info.payload_type;
397 return static_cast<int32_t>(encode_buffer_.size());
398}
399
400/////////////////////////////////////////
401// Sender
402//
403
kwibergc13ded52016-06-17 13:00:45404void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 18:57:10405 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 13:00:45406 rtc::CritScope lock(&acm_crit_sect_);
kwibergc13ded52016-06-17 13:00:45407 modifier(&encoder_stack_);
408}
409
kwibergc13ded52016-06-17 13:00:45410// Register a transport callback which will be called to deliver
411// the encoded buffers.
412int AudioCodingModuleImpl::RegisterTransportCallback(
413 AudioPacketizationCallback* transport) {
414 rtc::CritScope lock(&callback_crit_sect_);
415 packetization_callback_ = transport;
416 return 0;
417}
418
419// Add 10MS of raw (PCM) audio data to the encoder.
420int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
kwibergc13ded52016-06-17 13:00:45421 rtc::CritScope lock(&acm_crit_sect_);
Per Åhgren4f2e9402019-10-04 09:06:15422 int r = Add10MsDataInternal(audio_frame, &input_data_);
423 return r < 0 ? r : Encode(input_data_);
kwibergc13ded52016-06-17 13:00:45424}
425
426int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
427 InputData* input_data) {
428 if (audio_frame.samples_per_channel_ == 0) {
429 assert(false);
Mirko Bonadei675513b2017-11-09 10:09:25430 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 13:00:45431 return -1;
432 }
433
henrika33541572019-09-10 12:27:40434 if (audio_frame.sample_rate_hz_ > 192000) {
kwibergc13ded52016-06-17 13:00:45435 assert(false);
Mirko Bonadei675513b2017-11-09 10:09:25436 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 13:00:45437 return -1;
438 }
439
440 // If the length and frequency matches. We currently just support raw PCM.
441 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
442 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 10:09:25443 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 15:23:28444 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 13:00:45445 return -1;
446 }
447
Alex Loiko65438812019-02-22 09:13:44448 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2 &&
449 audio_frame.num_channels_ != 4 && audio_frame.num_channels_ != 6 &&
450 audio_frame.num_channels_ != 8) {
Mirko Bonadei675513b2017-11-09 10:09:25451 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 13:00:45452 return -1;
453 }
454
455 // Do we have a codec registered?
456 if (!HaveValidEncoder("Add10MsData")) {
457 return -1;
458 }
459
460 const AudioFrame* ptr_frame;
461 // Perform a resampling, also down-mix if it is required and can be
462 // performed before resampling (a down mix prior to resampling will take
463 // place if both primary and secondary encoders are mono and input is in
464 // stereo).
465 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
466 return -1;
467 }
468
469 // Check whether we need an up-mix or down-mix?
470 const size_t current_num_channels = encoder_stack_->NumChannels();
471 const bool same_num_channels =
472 ptr_frame->num_channels_ == current_num_channels;
473
yujo36b1a5f2017-06-12 19:45:32474 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 13:00:45475 input_data->input_timestamp = ptr_frame->timestamp_;
kwibergc13ded52016-06-17 13:00:45476 input_data->length_per_channel = ptr_frame->samples_per_channel_;
477 input_data->audio_channel = current_num_channels;
478
Per Åhgren4f2e9402019-10-04 09:06:15479 if (!same_num_channels) {
480 // Remixes the input frame to the output data and in the process resize the
481 // output data if needed.
482 ReMix(*ptr_frame, current_num_channels, &input_data->buffer);
483
484 // For pushing data to primary, point the |ptr_audio| to correct buffer.
485 input_data->audio = input_data->buffer.data();
486 RTC_DCHECK_GE(input_data->buffer.size(),
487 input_data->length_per_channel * input_data->audio_channel);
488 } else {
489 // When adding data to encoders this pointer is pointing to an audio buffer
490 // with correct number of channels.
491 input_data->audio = ptr_frame->data();
492 }
493
kwibergc13ded52016-06-17 13:00:45494 return 0;
495}
496
497// Perform a resampling and down-mix if required. We down-mix only if
498// encoder is mono and input is stereo. In case of dual-streaming, both
499// encoders has to be mono for down-mix to take place.
500// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
501// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 19:45:32502// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 13:00:45503int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
504 const AudioFrame** ptr_out) {
505 const bool resample =
506 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
507
508 // This variable is true if primary codec and secondary codec (if exists)
509 // are both mono and input is stereo.
510 // TODO(henrik.lundin): This condition should probably be
511 // in_frame.num_channels_ > encoder_stack_->NumChannels()
512 const bool down_mix =
513 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
514
515 if (!first_10ms_data_) {
516 expected_in_ts_ = in_frame.timestamp_;
517 expected_codec_ts_ = in_frame.timestamp_;
518 first_10ms_data_ = true;
519 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 10:09:25520 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
521 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 13:00:45522 expected_codec_ts_ +=
523 (in_frame.timestamp_ - expected_in_ts_) *
524 static_cast<uint32_t>(
525 static_cast<double>(encoder_stack_->SampleRateHz()) /
526 static_cast<double>(in_frame.sample_rate_hz_));
527 expected_in_ts_ = in_frame.timestamp_;
528 }
529
kwibergc13ded52016-06-17 13:00:45530 if (!down_mix && !resample) {
531 // No pre-processing is required.
ossu63fb95a2016-07-06 16:34:22532 if (expected_in_ts_ == expected_codec_ts_) {
533 // If we've never resampled, we can use the input frame as-is
534 *ptr_out = &in_frame;
535 } else {
536 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
537 // we'll have to make a copy of it.
538 preprocess_frame_.CopyFrom(in_frame);
539 preprocess_frame_.timestamp_ = expected_codec_ts_;
540 *ptr_out = &preprocess_frame_;
541 }
542
kwibergc13ded52016-06-17 13:00:45543 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
544 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 13:00:45545 return 0;
546 }
547
548 *ptr_out = &preprocess_frame_;
549 preprocess_frame_.num_channels_ = in_frame.num_channels_;
550 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 19:45:32551 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 13:00:45552 if (down_mix) {
553 // If a resampling is required the output of a down-mix is written into a
554 // local buffer, otherwise, it will be written to the output frame.
Yves Gerey665174f2018-06-19 13:03:05555 int16_t* dest_ptr_audio =
556 resample ? audio : preprocess_frame_.mutable_data();
Per Åhgren4f2e9402019-10-04 09:06:15557 DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio);
kwibergc13ded52016-06-17 13:00:45558 preprocess_frame_.num_channels_ = 1;
559 // Set the input of the resampler is the down-mixed signal.
560 src_ptr_audio = audio;
561 }
562
563 preprocess_frame_.timestamp_ = expected_codec_ts_;
564 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
565 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
566 // If it is required, we have to do a resampling.
567 if (resample) {
568 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 19:45:32569 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 13:00:45570
571 int samples_per_channel = resampler_.Resample10Msec(
572 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
573 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
574 dest_ptr_audio);
575
576 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 10:09:25577 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 13:00:45578 return -1;
579 }
580 preprocess_frame_.samples_per_channel_ =
581 static_cast<size_t>(samples_per_channel);
582 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
583 }
584
585 expected_codec_ts_ +=
586 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
587 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
588
589 return 0;
590}
591
592/////////////////////////////////////////
kwibergc13ded52016-06-17 13:00:45593// (FEC) Forward Error Correction (codec internal)
594//
595
kwibergc13ded52016-06-17 13:00:45596int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
597 rtc::CritScope lock(&acm_crit_sect_);
598 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 14:49:59599 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 13:00:45600 }
601 return 0;
602}
603
604/////////////////////////////////////////
kwibergc13ded52016-06-17 13:00:45605// Receiver
606//
607
608int AudioCodingModuleImpl::InitializeReceiver() {
609 rtc::CritScope lock(&acm_crit_sect_);
610 return InitializeReceiverSafe();
611}
612
613// Initialize receiver, resets codec database etc.
614int AudioCodingModuleImpl::InitializeReceiverSafe() {
615 // If the receiver is already initialized then we want to destroy any
616 // existing decoders. After a call to this function, we should have a clean
617 // start-up.
kwiberg6b19b562016-09-20 11:02:25618 if (receiver_initialized_)
619 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 13:00:45620 receiver_.FlushBuffers();
621
kwibergc13ded52016-06-17 13:00:45622 receiver_initialized_ = true;
623 return 0;
624}
625
kwiberg1c07c702017-03-27 14:15:49626void AudioCodingModuleImpl::SetReceiveCodecs(
627 const std::map<int, SdpAudioFormat>& codecs) {
628 rtc::CritScope lock(&acm_crit_sect_);
629 receiver_.SetCodecs(codecs);
630}
631
kwibergc13ded52016-06-17 13:00:45632// Incoming packet from network parsed and ready for decode.
633int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
634 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 14:21:47635 const RTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 14:38:01636 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 13:00:45637 return receiver_.InsertPacket(
638 rtp_header,
639 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
640}
641
kwibergc13ded52016-06-17 13:00:45642// Get 10 milliseconds of raw audio data to play out.
643// Automatic resample to the requested frequency.
644int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
645 AudioFrame* audio_frame,
646 bool* muted) {
647 // GetAudio always returns 10 ms, at the requested sample rate.
648 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 10:09:25649 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 13:00:45650 return -1;
651 }
kwibergc13ded52016-06-17 13:00:45652 return 0;
653}
654
kwibergc13ded52016-06-17 13:00:45655/////////////////////////////////////////
656// Statistics
657//
658
659// TODO(turajs) change the return value to void. Also change the corresponding
660// NetEq function.
661int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
662 receiver_.GetNetworkStatistics(statistics);
663 return 0;
664}
665
666int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 10:09:25667 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 13:00:45668 rtc::CritScope lock(&callback_crit_sect_);
669 vad_callback_ = vad_callback;
670 return 0;
671}
672
kwibergc13ded52016-06-17 13:00:45673bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
674 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 10:09:25675 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 13:00:45676 return false;
677 }
678 return true;
679}
680
ivoce1198e02017-09-08 15:13:19681ANAStats AudioCodingModuleImpl::GetANAStats() const {
682 rtc::CritScope lock(&acm_crit_sect_);
683 if (encoder_stack_)
684 return encoder_stack_->GetANAStats();
685 // If no encoder is set, return default stats.
686 return ANAStats();
687}
688
kwibergc13ded52016-06-17 13:00:45689} // namespace
690
Karl Wiberg5817d3d2018-04-06 08:06:42691AudioCodingModule::Config::Config(
692 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
693 : neteq_config(),
694 clock(Clock::GetRealTimeClock()),
695 decoder_factory(decoder_factory) {
kwiberg36a43882016-08-29 12:33:32696 // Post-decode VAD is disabled by default in NetEq, however, Audio
697 // Conference Mixer relies on VAD decisions and fails without them.
698 neteq_config.enable_post_decode_vad = true;
699}
700
701AudioCodingModule::Config::Config(const Config&) = default;
702AudioCodingModule::Config::~Config() = default;
703
Henrik Lundin64dad832015-05-11 10:44:23704AudioCodingModule* AudioCodingModule::Create(const Config& config) {
kwibergc13ded52016-06-17 13:00:45705 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26706}
707
turaj@webrtc.org7959e162013-09-12 18:30:26708} // namespace webrtc