blob: d3493f5c3dd437fdf4432669451267f7d8fbcf5a [file] [log] [blame]
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_mixer/frame_combiner.h"
#include <algorithm>
#include <array>
#include <functional>
#include "api/array_view.h"
#include "audio/utility/audio_frame_operations.h"
#include "common_audio/include/audio_util.h"
#include "modules/audio_mixer/audio_frame_manipulator.h"
#include "modules/audio_mixer/audio_mixer_impl.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/metrics.h"
namespace webrtc {
namespace {
// Stereo, 48 kHz, 10 ms.
constexpr int kMaximumAmountOfChannels = 2;
constexpr int kMaximumChannelSize = 48 * AudioMixerImpl::kFrameDurationInMs;
using OneChannelBuffer = std::array<float, kMaximumChannelSize>;
void SetAudioFrameFields(const std::vector<AudioFrame*>& mix_list,
size_t number_of_channels,
int sample_rate,
size_t number_of_streams,
AudioFrame* audio_frame_for_mixing) {
const size_t samples_per_channel = static_cast<size_t>(
(sample_rate * webrtc::AudioMixerImpl::kFrameDurationInMs) / 1000);
// TODO(minyue): Issue bugs.webrtc.org/3390.
// Audio frame timestamp. The 'timestamp_' field is set to dummy
// value '0', because it is only supported in the one channel case and
// is then updated in the helper functions.
audio_frame_for_mixing->UpdateFrame(
0, nullptr, samples_per_channel, sample_rate, AudioFrame::kUndefined,
AudioFrame::kVadUnknown, number_of_channels);
if (mix_list.empty()) {
audio_frame_for_mixing->elapsed_time_ms_ = -1;
} else if (mix_list.size() == 1) {
audio_frame_for_mixing->timestamp_ = mix_list[0]->timestamp_;
audio_frame_for_mixing->elapsed_time_ms_ = mix_list[0]->elapsed_time_ms_;
audio_frame_for_mixing->ntp_time_ms_ = mix_list[0]->ntp_time_ms_;
}
}
void MixFewFramesWithNoLimiter(const std::vector<AudioFrame*>& mix_list,
AudioFrame* audio_frame_for_mixing) {
if (mix_list.empty()) {
audio_frame_for_mixing->Mute();
return;
}
RTC_DCHECK_LE(mix_list.size(), 1);
std::copy(mix_list[0]->data(),
mix_list[0]->data() +
mix_list[0]->num_channels_ * mix_list[0]->samples_per_channel_,
audio_frame_for_mixing->mutable_data());
}
std::array<OneChannelBuffer, kMaximumAmountOfChannels> MixToFloatFrame(
const std::vector<AudioFrame*>& mix_list,
size_t samples_per_channel,
size_t number_of_channels) {
// Convert to FloatS16 and mix.
using OneChannelBuffer = std::array<float, kMaximumChannelSize>;
std::array<OneChannelBuffer, kMaximumAmountOfChannels> mixing_buffer{};
for (size_t i = 0; i < mix_list.size(); ++i) {
const AudioFrame* const frame = mix_list[i];
for (size_t j = 0; j < number_of_channels; ++j) {
for (size_t k = 0; k < samples_per_channel; ++k) {
mixing_buffer[j][k] += frame->data()[number_of_channels * k + j];
}
}
}
return mixing_buffer;
}
void RunLimiter(AudioFrameView<float> mixing_buffer_view, Limiter* limiter) {
const size_t sample_rate = mixing_buffer_view.samples_per_channel() * 1000 /
AudioMixerImpl::kFrameDurationInMs;
// TODO(alessiob): Avoid calling SetSampleRate every time.
limiter->SetSampleRate(sample_rate);
limiter->Process(mixing_buffer_view);
}
// Both interleaves and rounds.
void InterleaveToAudioFrame(AudioFrameView<const float> mixing_buffer_view,
AudioFrame* audio_frame_for_mixing) {
const size_t number_of_channels = mixing_buffer_view.num_channels();
const size_t samples_per_channel = mixing_buffer_view.samples_per_channel();
// Put data in the result frame.
for (size_t i = 0; i < number_of_channels; ++i) {
for (size_t j = 0; j < samples_per_channel; ++j) {
audio_frame_for_mixing->mutable_data()[number_of_channels * j + i] =
FloatS16ToS16(mixing_buffer_view.channel(i)[j]);
}
}
}
} // namespace
FrameCombiner::FrameCombiner(bool use_limiter)
: data_dumper_(new ApmDataDumper(0)),
limiter_(static_cast<size_t>(48000), data_dumper_.get(), "AudioMixer"),
use_limiter_(use_limiter) {}
FrameCombiner::~FrameCombiner() = default;
void FrameCombiner::Combine(const std::vector<AudioFrame*>& mix_list,
size_t number_of_channels,
int sample_rate,
size_t number_of_streams,
AudioFrame* audio_frame_for_mixing) {
RTC_DCHECK(audio_frame_for_mixing);
LogMixingStats(mix_list, sample_rate, number_of_streams);
SetAudioFrameFields(mix_list, number_of_channels, sample_rate,
number_of_streams, audio_frame_for_mixing);
const size_t samples_per_channel = static_cast<size_t>(
(sample_rate * webrtc::AudioMixerImpl::kFrameDurationInMs) / 1000);
for (const auto* frame : mix_list) {
RTC_DCHECK_EQ(samples_per_channel, frame->samples_per_channel_);
RTC_DCHECK_EQ(sample_rate, frame->sample_rate_hz_);
}
// The 'num_channels_' field of frames in 'mix_list' could be
// different from 'number_of_channels'.
for (auto* frame : mix_list) {
RemixFrame(number_of_channels, frame);
}
if (number_of_streams <= 1) {
MixFewFramesWithNoLimiter(mix_list, audio_frame_for_mixing);
return;
}
std::array<OneChannelBuffer, kMaximumAmountOfChannels> mixing_buffer =
MixToFloatFrame(mix_list, samples_per_channel, number_of_channels);
// Put float data in an AudioFrameView.
std::array<float*, kMaximumAmountOfChannels> channel_pointers{};
for (size_t i = 0; i < number_of_channels; ++i) {
channel_pointers[i] = &mixing_buffer[i][0];
}
AudioFrameView<float> mixing_buffer_view(
&channel_pointers[0], number_of_channels, samples_per_channel);
if (use_limiter_) {
RunLimiter(mixing_buffer_view, &limiter_);
}
InterleaveToAudioFrame(mixing_buffer_view, audio_frame_for_mixing);
}
void FrameCombiner::LogMixingStats(const std::vector<AudioFrame*>& mix_list,
int sample_rate,
size_t number_of_streams) const {
// Log every second.
uma_logging_counter_++;
if (uma_logging_counter_ > 1000 / AudioMixerImpl::kFrameDurationInMs) {
uma_logging_counter_ = 0;
RTC_HISTOGRAM_COUNTS_100("WebRTC.Audio.AudioMixer.NumIncomingStreams",
static_cast<int>(number_of_streams));
RTC_HISTOGRAM_ENUMERATION(
"WebRTC.Audio.AudioMixer.NumIncomingActiveStreams",
static_cast<int>(mix_list.size()),
AudioMixerImpl::kMaximumAmountOfMixedAudioSources);
using NativeRate = AudioProcessing::NativeRate;
static constexpr NativeRate native_rates[] = {
NativeRate::kSampleRate8kHz, NativeRate::kSampleRate16kHz,
NativeRate::kSampleRate32kHz, NativeRate::kSampleRate48kHz};
const auto* rate_position = std::lower_bound(
std::begin(native_rates), std::end(native_rates), sample_rate);
RTC_HISTOGRAM_ENUMERATION(
"WebRTC.Audio.AudioMixer.MixingRate",
std::distance(std::begin(native_rates), rate_position),
arraysize(native_rates));
}
}
} // namespace webrtc