Moved the AGC render sample queue into the audio processing module
Several subcomponents inside APM copy render audio from
the render side to the capture side using the same
pattern. Currently this is done independently for the
submodules.
This CL moves the the AGC functionality for this into
APM.
BUG=webrtc:5298, webrtc:6540
Review-Url: https://codereview.webrtc.org/2444283002
Cr-Commit-Position: refs/heads/master@{#14770}
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc
index ace788e..12b4c94 100644
--- a/webrtc/modules/audio_processing/audio_processing_impl.cc
+++ b/webrtc/modules/audio_processing/audio_processing_impl.cc
@@ -708,7 +708,6 @@
// getters that need the capture lock held when being called.
rtc::CritScope cs_capture(&crit_capture_);
EmptyQueuedRenderAudio();
- public_submodules_->gain_control->ReadQueuedRenderData();
if (!src || !dest) {
return kNullPointerError;
@@ -766,106 +765,143 @@
return kNoError;
}
-void AudioProcessingImpl::QueueRenderAudio(const AudioBuffer* audio) {
+void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) {
EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
- &float_render_queue_buffer_);
+ &aec_render_queue_buffer_);
RTC_DCHECK_GE(160u, audio->num_frames_per_band());
// Insert the samples into the queue.
- if (!float_render_signal_queue_->Insert(&float_render_queue_buffer_)) {
+ if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
// Retry the insert (should always work).
- bool result =
- float_render_signal_queue_->Insert(&float_render_queue_buffer_);
+ bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_);
RTC_DCHECK(result);
}
EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
- &int16_render_queue_buffer_);
+ &aecm_render_queue_buffer_);
// Insert the samples into the queue.
- if (!int16_render_signal_queue_->Insert(&int16_render_queue_buffer_)) {
+ if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
// Retry the insert (should always work).
- bool result =
- int16_render_signal_queue_->Insert(&int16_render_queue_buffer_);
+ bool result = aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
RTC_DCHECK(result);
}
+
+ if (!constants_.use_experimental_agc) {
+ GainControlImpl::PackRenderAudioBuffer(audio, &agc_render_queue_buffer_);
+ // Insert the samples into the queue.
+ if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
+ // The data queue is full and needs to be emptied.
+ EmptyQueuedRenderAudio();
+
+ // Retry the insert (should always work).
+ bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
+ RTC_DCHECK(result);
+ }
+ }
}
void AudioProcessingImpl::AllocateRenderQueue() {
- const size_t new_float_render_queue_element_max_size =
+ const size_t new_aec_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerFrame *
EchoCancellationImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
- const size_t new_int16_render_queue_element_max_size =
+ const size_t new_aecm_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerFrame *
EchoControlMobileImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
+ const size_t new_agc_render_queue_element_max_size =
+ std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
+
// Reallocate the queues if the queue item sizes are too small to fit the
// data to put in the queues.
- if (float_render_queue_element_max_size_ <
- new_float_render_queue_element_max_size) {
- float_render_queue_element_max_size_ =
- new_float_render_queue_element_max_size;
+ if (aec_render_queue_element_max_size_ <
+ new_aec_render_queue_element_max_size) {
+ aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
std::vector<float> template_queue_element(
- float_render_queue_element_max_size_);
+ aec_render_queue_element_max_size_);
- float_render_signal_queue_.reset(
+ aec_render_signal_queue_.reset(
new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<float>(
- float_render_queue_element_max_size_)));
+ aec_render_queue_element_max_size_)));
- float_render_queue_buffer_.resize(float_render_queue_element_max_size_);
- float_capture_queue_buffer_.resize(float_render_queue_element_max_size_);
+ aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
+ aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
} else {
- float_render_signal_queue_->Clear();
+ aec_render_signal_queue_->Clear();
}
- if (int16_render_queue_element_max_size_ <
- new_int16_render_queue_element_max_size) {
- int16_render_queue_element_max_size_ =
- new_int16_render_queue_element_max_size;
+ if (aecm_render_queue_element_max_size_ <
+ new_aecm_render_queue_element_max_size) {
+ aecm_render_queue_element_max_size_ =
+ new_aecm_render_queue_element_max_size;
std::vector<int16_t> template_queue_element(
- int16_render_queue_element_max_size_);
+ aecm_render_queue_element_max_size_);
- int16_render_signal_queue_.reset(
+ aecm_render_signal_queue_.reset(
new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<int16_t>(
- int16_render_queue_element_max_size_)));
+ aecm_render_queue_element_max_size_)));
- int16_render_queue_buffer_.resize(int16_render_queue_element_max_size_);
- int16_capture_queue_buffer_.resize(int16_render_queue_element_max_size_);
+ aecm_render_queue_buffer_.resize(aecm_render_queue_element_max_size_);
+ aecm_capture_queue_buffer_.resize(aecm_render_queue_element_max_size_);
} else {
- int16_render_signal_queue_->Clear();
+ aecm_render_signal_queue_->Clear();
+ }
+
+ if (agc_render_queue_element_max_size_ <
+ new_agc_render_queue_element_max_size) {
+ agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
+
+ std::vector<int16_t> template_queue_element(
+ agc_render_queue_element_max_size_);
+
+ agc_render_signal_queue_.reset(
+ new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
+ kMaxNumFramesToBuffer, template_queue_element,
+ RenderQueueItemVerifier<int16_t>(
+ agc_render_queue_element_max_size_)));
+
+ agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
+ agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
+ } else {
+ agc_render_signal_queue_->Clear();
}
}
void AudioProcessingImpl::EmptyQueuedRenderAudio() {
rtc::CritScope cs_capture(&crit_capture_);
- while (float_render_signal_queue_->Remove(&float_capture_queue_buffer_)) {
+ while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) {
public_submodules_->echo_cancellation->ProcessRenderAudio(
- float_capture_queue_buffer_);
+ aec_capture_queue_buffer_);
}
- while (int16_render_signal_queue_->Remove(&int16_capture_queue_buffer_)) {
+ while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
public_submodules_->echo_control_mobile->ProcessRenderAudio(
- int16_capture_queue_buffer_);
+ aecm_capture_queue_buffer_);
+ }
+
+ while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
+ public_submodules_->gain_control->ProcessRenderAudio(
+ agc_capture_queue_buffer_);
}
}
@@ -880,7 +916,6 @@
// as well.
rtc::CritScope cs_capture(&crit_capture_);
EmptyQueuedRenderAudio();
- public_submodules_->gain_control->ReadQueuedRenderData();
}
if (!frame) {
@@ -1242,10 +1277,6 @@
#endif
QueueRenderAudio(render_buffer);
- if (!constants_.use_experimental_agc) {
- RETURN_ON_ERR(
- public_submodules_->gain_control->ProcessRenderAudio(render_buffer));
- }
if (submodule_states_.RenderMultiBandProcessingActive() &&
SampleRateSupportsMultiBand(
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h
index 817b6b2..aec4ed7 100644
--- a/webrtc/modules/audio_processing/audio_processing_impl.h
+++ b/webrtc/modules/audio_processing/audio_processing_impl.h
@@ -238,7 +238,7 @@
void EmptyQueuedRenderAudio();
void AllocateRenderQueue()
EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
- void QueueRenderAudio(const AudioBuffer* audio)
+ void QueueRenderAudio(AudioBuffer* audio)
EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
// Capture-side exclusive methods possibly running APM in a multi-threaded
@@ -371,22 +371,30 @@
std::unique_ptr<AudioBuffer> render_audio;
} render_ GUARDED_BY(crit_render_);
- size_t float_render_queue_element_max_size_ GUARDED_BY(crit_render_)
+ size_t aec_render_queue_element_max_size_ GUARDED_BY(crit_render_)
GUARDED_BY(crit_capture_) = 0;
- std::vector<float> float_render_queue_buffer_ GUARDED_BY(crit_render_);
- std::vector<float> float_capture_queue_buffer_ GUARDED_BY(crit_capture_);
+ std::vector<float> aec_render_queue_buffer_ GUARDED_BY(crit_render_);
+ std::vector<float> aec_capture_queue_buffer_ GUARDED_BY(crit_capture_);
- size_t int16_render_queue_element_max_size_ GUARDED_BY(crit_render_)
+ size_t aecm_render_queue_element_max_size_ GUARDED_BY(crit_render_)
GUARDED_BY(crit_capture_) = 0;
- std::vector<int16_t> int16_render_queue_buffer_ GUARDED_BY(crit_render_);
- std::vector<int16_t> int16_capture_queue_buffer_ GUARDED_BY(crit_capture_);
+ std::vector<int16_t> aecm_render_queue_buffer_ GUARDED_BY(crit_render_);
+ std::vector<int16_t> aecm_capture_queue_buffer_ GUARDED_BY(crit_capture_);
+
+ size_t agc_render_queue_element_max_size_ GUARDED_BY(crit_render_)
+ GUARDED_BY(crit_capture_) = 0;
+ std::vector<int16_t> agc_render_queue_buffer_ GUARDED_BY(crit_render_);
+ std::vector<int16_t> agc_capture_queue_buffer_ GUARDED_BY(crit_capture_);
// Lock protection not needed.
std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
- float_render_signal_queue_;
+ aec_render_signal_queue_;
std::unique_ptr<
SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
- int16_render_signal_queue_;
+ aecm_render_signal_queue_;
+ std::unique_ptr<
+ SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
+ agc_render_signal_queue_;
};
} // namespace webrtc
diff --git a/webrtc/modules/audio_processing/gain_control_impl.cc b/webrtc/modules/audio_processing/gain_control_impl.cc
index aa4316d..8f707fc 100644
--- a/webrtc/modules/audio_processing/gain_control_impl.cc
+++ b/webrtc/modules/audio_processing/gain_control_impl.cc
@@ -33,13 +33,6 @@
return -1;
}
-// Maximum length that a frame of samples can have.
-static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160;
-// Maximum number of frames to buffer in the render queue.
-// TODO(peah): Decrease this once we properly handle hugely unbalanced
-// reverse and forward call numbers.
-static const size_t kMaxNumFramesToBuffer = 100;
-
} // namespace
class GainControlImpl::GainController {
@@ -103,74 +96,37 @@
compression_gain_db_(9),
analog_capture_level_(0),
was_analog_level_set_(false),
- stream_is_saturated_(false),
- render_queue_element_max_size_(0) {
+ stream_is_saturated_(false) {
RTC_DCHECK(crit_render);
RTC_DCHECK(crit_capture);
}
GainControlImpl::~GainControlImpl() {}
-int GainControlImpl::ProcessRenderAudio(AudioBuffer* audio) {
- rtc::CritScope cs(crit_render_);
- if (!enabled_) {
- return AudioProcessing::kNoError;
- }
-
- RTC_DCHECK_GE(160u, audio->num_frames_per_band());
-
- render_queue_buffer_.resize(0);
- for (auto& gain_controller : gain_controllers_) {
- int err = WebRtcAgc_GetAddFarendError(gain_controller->state(),
- audio->num_frames_per_band());
-
- if (err != AudioProcessing::kNoError) {
- return AudioProcessing::kUnspecifiedError;
- }
-
- // Buffer the samples in the render queue.
- render_queue_buffer_.insert(
- render_queue_buffer_.end(), audio->mixed_low_pass_data(),
- (audio->mixed_low_pass_data() + audio->num_frames_per_band()));
- }
-
- // Insert the samples into the queue.
- if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
- // The data queue is full and needs to be emptied.
- ReadQueuedRenderData();
-
- // Retry the insert (should always work).
- RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true);
- }
-
- return AudioProcessing::kNoError;
-}
-
-// Read chunks of data that were received and queued on the render side from
-// a queue. All the data chunks are buffered into the farend signal of the AGC.
-void GainControlImpl::ReadQueuedRenderData() {
- rtc::CritScope cs(crit_capture_);
-
+void GainControlImpl::ProcessRenderAudio(
+ rtc::ArrayView<const int16_t> packed_render_audio) {
+ rtc::CritScope cs_capture(crit_capture_);
if (!enabled_) {
return;
}
- while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
- size_t buffer_index = 0;
- RTC_DCHECK(num_proc_channels_);
- RTC_DCHECK_LT(0ul, *num_proc_channels_);
- const size_t num_frames_per_band =
- capture_queue_buffer_.size() / (*num_proc_channels_);
- for (auto& gain_controller : gain_controllers_) {
- WebRtcAgc_AddFarend(gain_controller->state(),
- &capture_queue_buffer_[buffer_index],
- num_frames_per_band);
-
- buffer_index += num_frames_per_band;
- }
+ for (auto& gain_controller : gain_controllers_) {
+ WebRtcAgc_AddFarend(gain_controller->state(), packed_render_audio.data(),
+ packed_render_audio.size());
}
}
+void GainControlImpl::PackRenderAudioBuffer(
+ AudioBuffer* audio,
+ std::vector<int16_t>* packed_buffer) {
+ RTC_DCHECK_GE(160u, audio->num_frames_per_band());
+
+ packed_buffer->clear();
+ packed_buffer->insert(
+ packed_buffer->end(), audio->mixed_low_pass_data(),
+ (audio->mixed_low_pass_data() + audio->num_frames_per_band()));
+}
+
int GainControlImpl::AnalyzeCaptureAudio(AudioBuffer* audio) {
rtc::CritScope cs(crit_capture_);
@@ -447,33 +403,6 @@
}
Configure();
-
- AllocateRenderQueue();
-}
-
-void GainControlImpl::AllocateRenderQueue() {
- rtc::CritScope cs_render(crit_render_);
- rtc::CritScope cs_capture(crit_capture_);
-
- RTC_DCHECK(num_proc_channels_);
- const size_t new_render_queue_element_max_size = std::max<size_t>(
- static_cast<size_t>(1),
- kMaxAllowedValuesOfSamplesPerFrame * (*num_proc_channels_));
-
- if (render_queue_element_max_size_ < new_render_queue_element_max_size) {
- render_queue_element_max_size_ = new_render_queue_element_max_size;
- std::vector<int16_t> template_queue_element(render_queue_element_max_size_);
-
- render_signal_queue_.reset(
- new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
- kMaxNumFramesToBuffer, template_queue_element,
- RenderQueueItemVerifier<int16_t>(render_queue_element_max_size_)));
-
- render_queue_buffer_.resize(render_queue_element_max_size_);
- capture_queue_buffer_.resize(render_queue_element_max_size_);
- } else {
- render_signal_queue_->Clear();
- }
}
int GainControlImpl::Configure() {
diff --git a/webrtc/modules/audio_processing/gain_control_impl.h b/webrtc/modules/audio_processing/gain_control_impl.h
index 1af6d7b..812b88c 100644
--- a/webrtc/modules/audio_processing/gain_control_impl.h
+++ b/webrtc/modules/audio_processing/gain_control_impl.h
@@ -31,21 +31,21 @@
rtc::CriticalSection* crit_capture);
~GainControlImpl() override;
- int ProcessRenderAudio(AudioBuffer* audio);
+ void ProcessRenderAudio(rtc::ArrayView<const int16_t> packed_render_audio);
int AnalyzeCaptureAudio(AudioBuffer* audio);
int ProcessCaptureAudio(AudioBuffer* audio, bool stream_has_echo);
void Initialize(size_t num_proc_channels, int sample_rate_hz);
+ static void PackRenderAudioBuffer(AudioBuffer* audio,
+ std::vector<int16_t>* packed_buffer);
+
// GainControl implementation.
bool is_enabled() const override;
int stream_analog_level() override;
bool is_limiter_enabled() const override;
Mode mode() const override;
- // Reads render side data that has been queued on the render call.
- void ReadQueuedRenderData();
-
int compression_gain_db() const override;
private:
@@ -64,7 +64,6 @@
int analog_level_maximum() const override;
bool stream_is_saturated() const override;
- void AllocateRenderQueue();
int Configure();
rtc::CriticalSection* const crit_render_ ACQUIRED_BEFORE(crit_capture_);
@@ -82,16 +81,6 @@
bool was_analog_level_set_ GUARDED_BY(crit_capture_);
bool stream_is_saturated_ GUARDED_BY(crit_capture_);
- size_t render_queue_element_max_size_ GUARDED_BY(crit_render_)
- GUARDED_BY(crit_capture_);
- std::vector<int16_t> render_queue_buffer_ GUARDED_BY(crit_render_);
- std::vector<int16_t> capture_queue_buffer_ GUARDED_BY(crit_capture_);
-
- // Lock protection not needed.
- std::unique_ptr<
- SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
- render_signal_queue_;
-
std::vector<std::unique_ptr<GainController>> gain_controllers_;
rtc::Optional<size_t> num_proc_channels_ GUARDED_BY(crit_capture_);
diff --git a/webrtc/modules/audio_processing/gain_control_unittest.cc b/webrtc/modules/audio_processing/gain_control_unittest.cc
index 989771a..1c89d76 100644
--- a/webrtc/modules/audio_processing/gain_control_unittest.cc
+++ b/webrtc/modules/audio_processing/gain_control_unittest.cc
@@ -30,8 +30,9 @@
capture_audio_buffer->SplitIntoFrequencyBands();
}
- gain_controller->ProcessRenderAudio(render_audio_buffer);
- gain_controller->ReadQueuedRenderData();
+ std::vector<int16_t> render_audio;
+ GainControlImpl::PackRenderAudioBuffer(render_audio_buffer, &render_audio);
+ gain_controller->ProcessRenderAudio(render_audio);
gain_controller->AnalyzeCaptureAudio(capture_audio_buffer);
gain_controller->ProcessCaptureAudio(capture_audio_buffer, false);
