Move dynamic memory allocations of webrtc::AudioMixerImpl from RT thead
(4 vector allocations removed)
Bug: webrtc:12035,webrtc:12036
Change-Id: Ie0d734cd0016a27c57809af67187ceb97f92f233
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/188621
Commit-Queue: Olga Sharonova <olka@webrtc.org>
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32441}
diff --git a/modules/audio_mixer/audio_mixer_impl.cc b/modules/audio_mixer/audio_mixer_impl.cc
index 6552953..04a8bcf 100644
--- a/modules/audio_mixer/audio_mixer_impl.cc
+++ b/modules/audio_mixer/audio_mixer_impl.cc
@@ -24,9 +24,23 @@
#include "rtc_base/ref_counted_object.h"
namespace webrtc {
+
+struct AudioMixerImpl::SourceStatus {
+ SourceStatus(Source* audio_source, bool is_mixed, float gain)
+ : audio_source(audio_source), is_mixed(is_mixed), gain(gain) {}
+ Source* audio_source = nullptr;
+ bool is_mixed = false;
+ float gain = 0.0f;
+
+ // A frame that will be passed to audio_source->GetAudioFrameWithInfo.
+ AudioFrame audio_frame;
+};
+
namespace {
struct SourceFrame {
+ SourceFrame() = default;
+
SourceFrame(AudioMixerImpl::SourceStatus* source_status,
AudioFrame* audio_frame,
bool muted)
@@ -57,6 +71,7 @@
};
// ShouldMixBefore(a, b) is used to select mixer sources.
+// Returns true if `a` is preferred over `b` as a source to be mixed.
bool ShouldMixBefore(const SourceFrame& a, const SourceFrame& b) {
if (a.muted != b.muted) {
return b.muted;
@@ -73,7 +88,7 @@
}
void RampAndUpdateGain(
- const std::vector<SourceFrame>& mixed_sources_and_frames) {
+ rtc::ArrayView<const SourceFrame> mixed_sources_and_frames) {
for (const auto& source_frame : mixed_sources_and_frames) {
float target_gain = source_frame.source_status->is_mixed ? 1.0f : 0.0f;
Ramp(source_frame.source_status->gain, target_gain,
@@ -82,9 +97,11 @@
}
}
-AudioMixerImpl::SourceStatusList::const_iterator FindSourceInList(
+std::vector<std::unique_ptr<AudioMixerImpl::SourceStatus>>::const_iterator
+FindSourceInList(
AudioMixerImpl::Source const* audio_source,
- AudioMixerImpl::SourceStatusList const* audio_source_list) {
+ std::vector<std::unique_ptr<AudioMixerImpl::SourceStatus>> const*
+ audio_source_list) {
return std::find_if(
audio_source_list->begin(), audio_source_list->end(),
[audio_source](const std::unique_ptr<AudioMixerImpl::SourceStatus>& p) {
@@ -93,14 +110,31 @@
}
} // namespace
+struct AudioMixerImpl::HelperContainers {
+ void resize(size_t size) {
+ audio_to_mix.resize(size);
+ audio_source_mixing_data_list.resize(size);
+ ramp_list.resize(size);
+ preferred_rates.resize(size);
+ }
+
+ std::vector<AudioFrame*> audio_to_mix;
+ std::vector<SourceFrame> audio_source_mixing_data_list;
+ std::vector<SourceFrame> ramp_list;
+ std::vector<int> preferred_rates;
+};
+
AudioMixerImpl::AudioMixerImpl(
std::unique_ptr<OutputRateCalculator> output_rate_calculator,
bool use_limiter)
: output_rate_calculator_(std::move(output_rate_calculator)),
- output_frequency_(0),
- sample_size_(0),
audio_source_list_(),
- frame_combiner_(use_limiter) {}
+ helper_containers_(std::make_unique<HelperContainers>()),
+ frame_combiner_(use_limiter) {
+ const int kTypicalMaxNumberOfMixedStreams = 3;
+ audio_source_list_.reserve(kTypicalMaxNumberOfMixedStreams);
+ helper_containers_->resize(kTypicalMaxNumberOfMixedStreams);
+}
AudioMixerImpl::~AudioMixerImpl() {}
@@ -121,40 +155,23 @@
void AudioMixerImpl::Mix(size_t number_of_channels,
AudioFrame* audio_frame_for_mixing) {
RTC_DCHECK(number_of_channels >= 1);
- RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
-
- CalculateOutputFrequency();
-
- {
- MutexLock lock(&mutex_);
- const size_t number_of_streams = audio_source_list_.size();
- frame_combiner_.Combine(GetAudioFromSources(), number_of_channels,
- OutputFrequency(), number_of_streams,
- audio_frame_for_mixing);
- }
-
- return;
-}
-
-void AudioMixerImpl::CalculateOutputFrequency() {
- RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
MutexLock lock(&mutex_);
- std::vector<int> preferred_rates;
+ size_t number_of_streams = audio_source_list_.size();
+
std::transform(audio_source_list_.begin(), audio_source_list_.end(),
- std::back_inserter(preferred_rates),
+ helper_containers_->preferred_rates.begin(),
[&](std::unique_ptr<SourceStatus>& a) {
return a->audio_source->PreferredSampleRate();
});
- output_frequency_ =
- output_rate_calculator_->CalculateOutputRate(preferred_rates);
- sample_size_ = (output_frequency_ * kFrameDurationInMs) / 1000;
-}
+ int output_frequency = output_rate_calculator_->CalculateOutputRateFromRange(
+ rtc::ArrayView<const int>(helper_containers_->preferred_rates.data(),
+ number_of_streams));
-int AudioMixerImpl::OutputFrequency() const {
- RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
- return output_frequency_;
+ frame_combiner_.Combine(GetAudioFromSources(output_frequency),
+ number_of_channels, output_frequency,
+ number_of_streams, audio_frame_for_mixing);
}
bool AudioMixerImpl::AddSource(Source* audio_source) {
@@ -164,6 +181,7 @@
audio_source_list_.end())
<< "Source already added to mixer";
audio_source_list_.emplace_back(new SourceStatus(audio_source, false, 0));
+ helper_containers_->resize(audio_source_list_.size());
return true;
}
@@ -175,35 +193,37 @@
audio_source_list_.erase(iter);
}
-AudioFrameList AudioMixerImpl::GetAudioFromSources() {
- RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
- AudioFrameList result;
- std::vector<SourceFrame> audio_source_mixing_data_list;
- std::vector<SourceFrame> ramp_list;
-
+rtc::ArrayView<AudioFrame* const> AudioMixerImpl::GetAudioFromSources(
+ int output_frequency) {
// Get audio from the audio sources and put it in the SourceFrame vector.
+ int audio_source_mixing_data_count = 0;
for (auto& source_and_status : audio_source_list_) {
const auto audio_frame_info =
source_and_status->audio_source->GetAudioFrameWithInfo(
- OutputFrequency(), &source_and_status->audio_frame);
+ output_frequency, &source_and_status->audio_frame);
if (audio_frame_info == Source::AudioFrameInfo::kError) {
RTC_LOG_F(LS_WARNING) << "failed to GetAudioFrameWithInfo() from source";
continue;
}
- audio_source_mixing_data_list.emplace_back(
- source_and_status.get(), &source_and_status->audio_frame,
- audio_frame_info == Source::AudioFrameInfo::kMuted);
+ helper_containers_
+ ->audio_source_mixing_data_list[audio_source_mixing_data_count++] =
+ SourceFrame(source_and_status.get(), &source_and_status->audio_frame,
+ audio_frame_info == Source::AudioFrameInfo::kMuted);
}
+ rtc::ArrayView<SourceFrame> audio_source_mixing_data_view(
+ helper_containers_->audio_source_mixing_data_list.data(),
+ audio_source_mixing_data_count);
// Sort frames by sorting function.
- std::sort(audio_source_mixing_data_list.begin(),
- audio_source_mixing_data_list.end(), ShouldMixBefore);
+ std::sort(audio_source_mixing_data_view.begin(),
+ audio_source_mixing_data_view.end(), ShouldMixBefore);
int max_audio_frame_counter = kMaximumAmountOfMixedAudioSources;
-
+ int ramp_list_lengh = 0;
+ int audio_to_mix_count = 0;
// Go through list in order and put unmuted frames in result list.
- for (const auto& p : audio_source_mixing_data_list) {
+ for (const auto& p : audio_source_mixing_data_view) {
// Filter muted.
if (p.muted) {
p.source_status->is_mixed = false;
@@ -214,19 +234,21 @@
bool is_mixed = false;
if (max_audio_frame_counter > 0) {
--max_audio_frame_counter;
- result.push_back(p.audio_frame);
- ramp_list.emplace_back(p.source_status, p.audio_frame, false, -1);
+ helper_containers_->audio_to_mix[audio_to_mix_count++] = p.audio_frame;
+ helper_containers_->ramp_list[ramp_list_lengh++] =
+ SourceFrame(p.source_status, p.audio_frame, false, -1);
is_mixed = true;
}
p.source_status->is_mixed = is_mixed;
}
- RampAndUpdateGain(ramp_list);
- return result;
+ RampAndUpdateGain(rtc::ArrayView<SourceFrame>(
+ helper_containers_->ramp_list.data(), ramp_list_lengh));
+ return rtc::ArrayView<AudioFrame* const>(
+ helper_containers_->audio_to_mix.data(), audio_to_mix_count);
}
bool AudioMixerImpl::GetAudioSourceMixabilityStatusForTest(
AudioMixerImpl::Source* audio_source) const {
- RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
MutexLock lock(&mutex_);
const auto iter = FindSourceInList(audio_source, &audio_source_list_);
diff --git a/modules/audio_mixer/audio_mixer_impl.h b/modules/audio_mixer/audio_mixer_impl.h
index 57b1f5e..0a13082 100644
--- a/modules/audio_mixer/audio_mixer_impl.h
+++ b/modules/audio_mixer/audio_mixer_impl.h
@@ -16,6 +16,7 @@
#include <memory>
#include <vector>
+#include "api/array_view.h"
#include "api/audio/audio_frame.h"
#include "api/audio/audio_mixer.h"
#include "api/scoped_refptr.h"
@@ -28,22 +29,9 @@
namespace webrtc {
-typedef std::vector<AudioFrame*> AudioFrameList;
-
class AudioMixerImpl : public AudioMixer {
public:
- struct SourceStatus {
- SourceStatus(Source* audio_source, bool is_mixed, float gain)
- : audio_source(audio_source), is_mixed(is_mixed), gain(gain) {}
- Source* audio_source = nullptr;
- bool is_mixed = false;
- float gain = 0.0f;
-
- // A frame that will be passed to audio_source->GetAudioFrameWithInfo.
- AudioFrame audio_frame;
- };
-
- using SourceStatusList = std::vector<std::unique_ptr<SourceStatus>>;
+ struct SourceStatus;
// AudioProcessing only accepts 10 ms frames.
static const int kFrameDurationInMs = 10;
@@ -75,32 +63,29 @@
bool use_limiter);
private:
- // Set mixing frequency through OutputFrequencyCalculator.
- void CalculateOutputFrequency();
- // Get mixing frequency.
- int OutputFrequency() const;
+ struct HelperContainers;
// Compute what audio sources to mix from audio_source_list_. Ramp
// in and out. Update mixed status. Mixes up to
// kMaximumAmountOfMixedAudioSources audio sources.
- AudioFrameList GetAudioFromSources() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ rtc::ArrayView<AudioFrame* const> GetAudioFromSources(int output_frequency)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// The critical section lock guards audio source insertion and
// removal, which can be done from any thread. The race checker
// checks that mixing is done sequentially.
mutable Mutex mutex_;
- rtc::RaceChecker race_checker_;
std::unique_ptr<OutputRateCalculator> output_rate_calculator_;
- // The current sample frequency and sample size when mixing.
- int output_frequency_ RTC_GUARDED_BY(race_checker_);
- size_t sample_size_ RTC_GUARDED_BY(race_checker_);
- // List of all audio sources. Note all lists are disjunct
- SourceStatusList audio_source_list_ RTC_GUARDED_BY(mutex_); // May be mixed.
+ // List of all audio sources.
+ std::vector<std::unique_ptr<SourceStatus>> audio_source_list_
+ RTC_GUARDED_BY(mutex_);
+ const std::unique_ptr<HelperContainers> helper_containers_
+ RTC_GUARDED_BY(mutex_);
// Component that handles actual adding of audio frames.
- FrameCombiner frame_combiner_ RTC_GUARDED_BY(race_checker_);
+ FrameCombiner frame_combiner_;
RTC_DISALLOW_COPY_AND_ASSIGN(AudioMixerImpl);
};
diff --git a/modules/audio_mixer/audio_mixer_impl_unittest.cc b/modules/audio_mixer/audio_mixer_impl_unittest.cc
index 383771c..18a4384 100644
--- a/modules/audio_mixer/audio_mixer_impl_unittest.cc
+++ b/modules/audio_mixer/audio_mixer_impl_unittest.cc
@@ -105,7 +105,8 @@
class CustomRateCalculator : public OutputRateCalculator {
public:
explicit CustomRateCalculator(int rate) : rate_(rate) {}
- int CalculateOutputRate(const std::vector<int>& preferred_rates) override {
+ int CalculateOutputRateFromRange(
+ rtc::ArrayView<const int> preferred_rates) override {
return rate_;
}
@@ -598,8 +599,8 @@
class HighOutputRateCalculator : public OutputRateCalculator {
public:
static const int kDefaultFrequency = 76000;
- int CalculateOutputRate(
- const std::vector<int>& preferred_sample_rates) override {
+ int CalculateOutputRateFromRange(
+ rtc::ArrayView<const int> preferred_sample_rates) override {
return kDefaultFrequency;
}
~HighOutputRateCalculator() override {}
diff --git a/modules/audio_mixer/default_output_rate_calculator.cc b/modules/audio_mixer/default_output_rate_calculator.cc
index 57d88b6..5f24b65 100644
--- a/modules/audio_mixer/default_output_rate_calculator.cc
+++ b/modules/audio_mixer/default_output_rate_calculator.cc
@@ -18,14 +18,14 @@
namespace webrtc {
-int DefaultOutputRateCalculator::CalculateOutputRate(
- const std::vector<int>& preferred_sample_rates) {
+int DefaultOutputRateCalculator::CalculateOutputRateFromRange(
+ rtc::ArrayView<const int> preferred_sample_rates) {
if (preferred_sample_rates.empty()) {
return DefaultOutputRateCalculator::kDefaultFrequency;
}
using NativeRate = AudioProcessing::NativeRate;
const int maximal_frequency = *std::max_element(
- preferred_sample_rates.begin(), preferred_sample_rates.end());
+ preferred_sample_rates.cbegin(), preferred_sample_rates.cend());
RTC_DCHECK_LE(NativeRate::kSampleRate8kHz, maximal_frequency);
RTC_DCHECK_GE(NativeRate::kSampleRate48kHz, maximal_frequency);
diff --git a/modules/audio_mixer/default_output_rate_calculator.h b/modules/audio_mixer/default_output_rate_calculator.h
index 2d34f34..a7aaf68 100644
--- a/modules/audio_mixer/default_output_rate_calculator.h
+++ b/modules/audio_mixer/default_output_rate_calculator.h
@@ -13,6 +13,7 @@
#include <vector>
+#include "api/array_view.h"
#include "modules/audio_mixer/output_rate_calculator.h"
namespace webrtc {
@@ -25,8 +26,8 @@
// sample rates. A native rate is one in
// AudioProcessing::NativeRate. If |preferred_sample_rates| is
// empty, returns |kDefaultFrequency|.
- int CalculateOutputRate(
- const std::vector<int>& preferred_sample_rates) override;
+ int CalculateOutputRateFromRange(
+ rtc::ArrayView<const int> preferred_sample_rates) override;
~DefaultOutputRateCalculator() override {}
};
diff --git a/modules/audio_mixer/frame_combiner.cc b/modules/audio_mixer/frame_combiner.cc
index f7c6a0c..e184506 100644
--- a/modules/audio_mixer/frame_combiner.cc
+++ b/modules/audio_mixer/frame_combiner.cc
@@ -35,7 +35,7 @@
std::array<std::array<float, FrameCombiner::kMaximumChannelSize>,
FrameCombiner::kMaximumNumberOfChannels>;
-void SetAudioFrameFields(const std::vector<AudioFrame*>& mix_list,
+void SetAudioFrameFields(rtc::ArrayView<const AudioFrame* const> mix_list,
size_t number_of_channels,
int sample_rate,
size_t number_of_streams,
@@ -61,7 +61,7 @@
}
}
-void MixFewFramesWithNoLimiter(const std::vector<AudioFrame*>& mix_list,
+void MixFewFramesWithNoLimiter(rtc::ArrayView<const AudioFrame* const> mix_list,
AudioFrame* audio_frame_for_mixing) {
if (mix_list.empty()) {
audio_frame_for_mixing->Mute();
@@ -74,7 +74,7 @@
audio_frame_for_mixing->mutable_data());
}
-void MixToFloatFrame(const std::vector<AudioFrame*>& mix_list,
+void MixToFloatFrame(rtc::ArrayView<const AudioFrame* const> mix_list,
size_t samples_per_channel,
size_t number_of_channels,
MixingBuffer* mixing_buffer) {
@@ -140,7 +140,7 @@
FrameCombiner::~FrameCombiner() = default;
-void FrameCombiner::Combine(const std::vector<AudioFrame*>& mix_list,
+void FrameCombiner::Combine(rtc::ArrayView<AudioFrame* const> mix_list,
size_t number_of_channels,
int sample_rate,
size_t number_of_streams,
@@ -195,9 +195,10 @@
InterleaveToAudioFrame(mixing_buffer_view, audio_frame_for_mixing);
}
-void FrameCombiner::LogMixingStats(const std::vector<AudioFrame*>& mix_list,
- int sample_rate,
- size_t number_of_streams) const {
+void FrameCombiner::LogMixingStats(
+ rtc::ArrayView<const AudioFrame* const> mix_list,
+ int sample_rate,
+ size_t number_of_streams) const {
// Log every second.
uma_logging_counter_++;
if (uma_logging_counter_ > 1000 / AudioMixerImpl::kFrameDurationInMs) {
diff --git a/modules/audio_mixer/frame_combiner.h b/modules/audio_mixer/frame_combiner.h
index d989d02..9ddf81e 100644
--- a/modules/audio_mixer/frame_combiner.h
+++ b/modules/audio_mixer/frame_combiner.h
@@ -14,6 +14,7 @@
#include <memory>
#include <vector>
+#include "api/array_view.h"
#include "api/audio/audio_frame.h"
#include "modules/audio_processing/agc2/limiter.h"
@@ -32,7 +33,7 @@
// because 'mix_list' can be empty. The parameter
// 'number_of_streams' is used for determining whether to pass the
// data through a limiter.
- void Combine(const std::vector<AudioFrame*>& mix_list,
+ void Combine(rtc::ArrayView<AudioFrame* const> mix_list,
size_t number_of_channels,
int sample_rate,
size_t number_of_streams,
@@ -46,7 +47,7 @@
kMaximumNumberOfChannels>;
private:
- void LogMixingStats(const std::vector<AudioFrame*>& mix_list,
+ void LogMixingStats(rtc::ArrayView<const AudioFrame* const> mix_list,
int sample_rate,
size_t number_of_streams) const;
diff --git a/modules/audio_mixer/output_rate_calculator.h b/modules/audio_mixer/output_rate_calculator.h
index cb3ca96..d32fb29 100644
--- a/modules/audio_mixer/output_rate_calculator.h
+++ b/modules/audio_mixer/output_rate_calculator.h
@@ -13,14 +13,29 @@
#include <vector>
+#include "api/array_view.h"
+
namespace webrtc {
// Decides the sample rate of a mixing iteration given the preferred
// sample rates of the sources.
class OutputRateCalculator {
public:
+ virtual int CalculateOutputRateFromRange(
+ rtc::ArrayView<const int> preferred_sample_rates) {
+ // TODO(olka): Temporary workaround to reslove client dependencies.
+ std::vector<int> sample_rates(preferred_sample_rates.cbegin(),
+ preferred_sample_rates.cend());
+ return CalculateOutputRate(sample_rates);
+ }
+
+ // TODO(olka) to be removed as soon as the clients are switched to
+ // CalculateOutputRateFromRange()
virtual int CalculateOutputRate(
- const std::vector<int>& preferred_sample_rates) = 0;
+ const std::vector<int>& preferred_sample_rates) {
+ return CalculateOutputRateFromRange(preferred_sample_rates);
+ }
+
virtual ~OutputRateCalculator() {}
};
