Only create AEC2 when needed
This CL ensures that the AEC2 is only created when needed.
The changes in the CL are bitexact when running AEC2 via
audioproc_f
Bug: webrtc:8671
Change-Id: I5f6d33e45a7031c69ac53098781635c415668e49
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/129740
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Commit-Queue: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27772}
diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc
index 4d24ee6..6950623 100644
--- a/modules/audio_processing/audio_processing_impl.cc
+++ b/modules/audio_processing/audio_processing_impl.cc
@@ -429,7 +429,6 @@
new rtc::RefCountedObject<ResidualEchoDetector>();
}
- private_submodules_->echo_cancellation.reset(new EchoCancellationImpl());
private_submodules_->echo_control_mobile.reset(new EchoControlMobileImpl());
// TODO(alessiob): Move the injected gain controller once injection is
// implemented.
@@ -548,15 +547,8 @@
formats_.api_format.output_stream().num_channels(),
formats_.api_format.output_stream().num_frames()));
- private_submodules_->echo_cancellation->Initialize(
- proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
- num_proc_channels());
AllocateRenderQueue();
- int success = private_submodules_->echo_cancellation->enable_metrics(true);
- RTC_DCHECK_EQ(0, success);
- success = private_submodules_->echo_cancellation->enable_delay_logging(true);
- RTC_DCHECK_EQ(0, success);
private_submodules_->echo_control_mobile->Initialize(
proc_split_sample_rate_hz(), num_reverse_channels(),
num_output_channels());
@@ -802,7 +794,12 @@
rtc::CritScope cs_render(&crit_render_);
rtc::CritScope cs_capture(&crit_capture_);
- private_submodules_->echo_cancellation->SetExtraOptions(config);
+ capture_nonlocked_.use_aec2_extended_filter =
+ config.Get<ExtendedFilter>().enabled;
+ capture_nonlocked_.use_aec2_delay_agnostic =
+ config.Get<DelayAgnostic>().enabled;
+ capture_nonlocked_.use_aec2_refined_adaptive_filter =
+ config.Get<RefinedAdaptiveFilter>().enabled;
if (capture_.transient_suppressor_enabled !=
config.Get<ExperimentalNs>().enabled) {
@@ -1035,13 +1032,16 @@
RTC_DCHECK_GE(160, audio->num_frames_per_band());
// Insert the samples into the queue.
- if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
- // The data queue is full and needs to be emptied.
- EmptyQueuedRenderAudio();
+ if (private_submodules_->echo_cancellation) {
+ RTC_DCHECK(aec_render_signal_queue_);
+ if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
+ // The data queue is full and needs to be emptied.
+ EmptyQueuedRenderAudio();
- // Retry the insert (should always work).
- bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_);
- RTC_DCHECK(result);
+ // Retry the insert (should always work).
+ bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_);
+ RTC_DCHECK(result);
+ }
}
EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
@@ -1087,12 +1087,6 @@
}
void AudioProcessingImpl::AllocateRenderQueue() {
- const size_t new_aec_render_queue_element_max_size =
- std::max(static_cast<size_t>(1),
- kMaxAllowedValuesOfSamplesPerBand *
- EchoCancellationImpl::NumCancellersRequired(
- num_output_channels(), num_reverse_channels()));
-
const size_t new_aecm_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerBand *
@@ -1107,25 +1101,6 @@
// Reallocate the queues if the queue item sizes are too small to fit the
// data to put in the queues.
- if (aec_render_queue_element_max_size_ <
- new_aec_render_queue_element_max_size) {
- aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
-
- std::vector<float> template_queue_element(
- aec_render_queue_element_max_size_);
-
- aec_render_signal_queue_.reset(
- new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
- kMaxNumFramesToBuffer, template_queue_element,
- RenderQueueItemVerifier<float>(
- aec_render_queue_element_max_size_)));
-
- aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
- aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
- } else {
- aec_render_signal_queue_->Clear();
- }
-
if (aecm_render_queue_element_max_size_ <
new_aecm_render_queue_element_max_size) {
aecm_render_queue_element_max_size_ =
@@ -1187,9 +1162,12 @@
void AudioProcessingImpl::EmptyQueuedRenderAudio() {
rtc::CritScope cs_capture(&crit_capture_);
- while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) {
- private_submodules_->echo_cancellation->ProcessRenderAudio(
- aec_capture_queue_buffer_);
+ if (private_submodules_->echo_cancellation) {
+ RTC_DCHECK(aec_render_signal_queue_);
+ while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) {
+ private_submodules_->echo_cancellation->ProcessRenderAudio(
+ aec_capture_queue_buffer_);
+ }
}
while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
@@ -1283,7 +1261,8 @@
// Ensure that not both the AEC and AECM are active at the same time.
// TODO(peah): Simplify once the public API Enable functions for these
// are moved to APM.
- RTC_DCHECK(!(private_submodules_->echo_cancellation->is_enabled() &&
+ RTC_DCHECK(!((private_submodules_->echo_cancellation &&
+ private_submodules_->echo_cancellation->is_enabled()) &&
private_submodules_->echo_control_mobile->is_enabled()));
MaybeUpdateHistograms();
@@ -1368,7 +1347,8 @@
// Ensure that the stream delay was set before the call to the
// AEC ProcessCaptureAudio function.
- if (private_submodules_->echo_cancellation->is_enabled() &&
+ if (private_submodules_->echo_cancellation &&
+ private_submodules_->echo_cancellation->is_enabled() &&
!private_submodules_->echo_controller && !was_stream_delay_set()) {
return AudioProcessing::kStreamParameterNotSetError;
}
@@ -1383,7 +1363,7 @@
private_submodules_->echo_controller->ProcessCapture(
capture_buffer, capture_.echo_path_gain_change);
- } else {
+ } else if (private_submodules_->echo_cancellation) {
RETURN_ON_ERR(private_submodules_->echo_cancellation->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
}
@@ -1402,7 +1382,8 @@
}
if (!(private_submodules_->echo_controller ||
- private_submodules_->echo_cancellation->is_enabled())) {
+ (private_submodules_->echo_cancellation &&
+ private_submodules_->echo_cancellation->is_enabled()))) {
RETURN_ON_ERR(private_submodules_->echo_control_mobile->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
}
@@ -1426,7 +1407,8 @@
// TODO(peah): Add reporting from AEC3 whether there is echo.
RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(
capture_buffer,
- private_submodules_->echo_cancellation->stream_has_echo()));
+ private_submodules_->echo_cancellation &&
+ private_submodules_->echo_cancellation->stream_has_echo()));
if (submodule_states_.CaptureMultiBandProcessingActive() &&
SampleRateSupportsMultiBand(
@@ -1756,20 +1738,6 @@
stats.echo_return_loss_enhancement =
ec_metrics.echo_return_loss_enhancement;
stats.delay_ms = ec_metrics.delay_ms;
- } else if (private_submodules_->echo_cancellation->GetMetrics(&metrics) ==
- Error::kNoError) {
- if (metrics.divergent_filter_fraction != -1.0f) {
- stats.divergent_filter_fraction =
- absl::optional<double>(metrics.divergent_filter_fraction);
- }
- if (metrics.echo_return_loss.instant != -100) {
- stats.echo_return_loss =
- absl::optional<double>(metrics.echo_return_loss.instant);
- }
- if (metrics.echo_return_loss_enhancement.instant != -100) {
- stats.echo_return_loss_enhancement =
- absl::optional<double>(metrics.echo_return_loss_enhancement.instant);
- }
}
if (config_.residual_echo_detector.enabled) {
RTC_DCHECK(private_submodules_->echo_detector);
@@ -1778,18 +1746,6 @@
stats.residual_echo_likelihood_recent_max =
ed_metrics.echo_likelihood_recent_max;
}
- int delay_median, delay_std;
- float fraction_poor_delays;
- if (private_submodules_->echo_cancellation->GetDelayMetrics(
- &delay_median, &delay_std, &fraction_poor_delays) ==
- Error::kNoError) {
- if (delay_median >= 0) {
- stats.delay_median_ms = absl::optional<int32_t>(delay_median);
- }
- if (delay_std >= 0) {
- stats.delay_standard_deviation_ms = absl::optional<int32_t>(delay_std);
- }
- }
return stats;
}
@@ -1826,7 +1782,8 @@
bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
return submodule_states_.Update(
config_.high_pass_filter.enabled,
- private_submodules_->echo_cancellation->is_enabled(),
+ private_submodules_->echo_cancellation &&
+ private_submodules_->echo_cancellation->is_enabled(),
private_submodules_->echo_control_mobile->is_enabled(),
config_.residual_echo_detector.enabled,
public_submodules_->noise_suppression->is_enabled(),
@@ -1872,20 +1829,82 @@
}
capture_nonlocked_.echo_controller_enabled = true;
- } else {
- private_submodules_->echo_cancellation->Enable(
- config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode);
- private_submodules_->echo_control_mobile->Enable(
- config_.echo_canceller.enabled && config_.echo_canceller.mobile_mode);
- private_submodules_->echo_cancellation->set_suppression_level(
- config_.echo_canceller.legacy_moderate_suppression_level
- ? EchoCancellationImpl::SuppressionLevel::kModerateSuppression
- : EchoCancellationImpl::SuppressionLevel::kHighSuppression);
-
- private_submodules_->echo_controller.reset();
- capture_nonlocked_.echo_controller_enabled = false;
+ private_submodules_->echo_cancellation.reset();
+ aec_render_signal_queue_.reset();
+ private_submodules_->echo_control_mobile->Enable(false);
+ return;
}
+
+ private_submodules_->echo_controller.reset();
+ capture_nonlocked_.echo_controller_enabled = false;
+
+ if (!config_.echo_canceller.enabled) {
+ private_submodules_->echo_cancellation.reset();
+ aec_render_signal_queue_.reset();
+ private_submodules_->echo_control_mobile->Enable(false);
+ return;
+ }
+
+ if (config_.echo_canceller.mobile_mode) {
+ // Create and activate AECM.
+ // TODO(peah): Add an on-demand creation of AECmobile similar to AEC2.
+ private_submodules_->echo_control_mobile->Enable(true);
+
+ private_submodules_->echo_cancellation.reset();
+ aec_render_signal_queue_.reset();
+ return;
+ }
+
+ // Create and activate AEC2.
+ private_submodules_->echo_control_mobile->Enable(false);
+ private_submodules_->echo_cancellation.reset(new EchoCancellationImpl());
+ private_submodules_->echo_cancellation->SetExtraOptions(
+ capture_nonlocked_.use_aec2_extended_filter,
+ capture_nonlocked_.use_aec2_delay_agnostic,
+ capture_nonlocked_.use_aec2_refined_adaptive_filter);
+
+ const size_t new_aec_render_queue_element_max_size =
+ std::max(static_cast<size_t>(1),
+ kMaxAllowedValuesOfSamplesPerBand *
+ EchoCancellationImpl::NumCancellersRequired(
+ num_output_channels(), num_reverse_channels()));
+
+ // Reallocate the queues if the queue item sizes are too small to fit the
+ // data to put in the queues.
+ if (aec_render_queue_element_max_size_ <
+ new_aec_render_queue_element_max_size ||
+ !aec_render_signal_queue_) {
+ aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
+
+ std::vector<float> template_queue_element(
+ aec_render_queue_element_max_size_);
+
+ aec_render_signal_queue_.reset(
+ new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
+ kMaxNumFramesToBuffer, template_queue_element,
+ RenderQueueItemVerifier<float>(
+ aec_render_queue_element_max_size_)));
+
+ aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
+ aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
+ } else {
+ aec_render_signal_queue_->Clear();
+ }
+
+ private_submodules_->echo_cancellation->Initialize(
+ proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
+ num_proc_channels());
+
+ private_submodules_->echo_cancellation->Enable(true);
+
+ private_submodules_->echo_cancellation->set_suppression_level(
+ config_.echo_canceller.legacy_moderate_suppression_level
+ ? EchoCancellationImpl::SuppressionLevel::kModerateSuppression
+ : EchoCancellationImpl::SuppressionLevel::kHighSuppression);
+
+ private_submodules_->echo_controller.reset();
+ capture_nonlocked_.echo_controller_enabled = false;
}
void AudioProcessingImpl::InitializeGainController2() {
@@ -1935,7 +1954,8 @@
void AudioProcessingImpl::MaybeUpdateHistograms() {
static const int kMinDiffDelayMs = 60;
- if (private_submodules_->echo_cancellation->is_enabled()) {
+ if (private_submodules_->echo_cancellation &&
+ private_submodules_->echo_cancellation->is_enabled()) {
// Activate delay_jumps_ counters if we know echo_cancellation is running.
// If a stream has echo we know that the echo_cancellation is in process.
if (capture_.stream_delay_jumps == -1 &&
@@ -2009,8 +2029,12 @@
if (!aec_dump_) {
return;
}
- std::string experiments_description =
- private_submodules_->echo_cancellation->GetExperimentsDescription();
+
+ std::string experiments_description = "";
+ if (private_submodules_->echo_cancellation) {
+ experiments_description +=
+ private_submodules_->echo_cancellation->GetExperimentsDescription();
+ }
// TODO(peah): Add semicolon-separated concatenations of experiment
// descriptions for other submodules.
if (constants_.agc_clipped_level_min != kClippedLevelMin) {
@@ -2027,13 +2051,19 @@
apm_config.aec_enabled = config_.echo_canceller.enabled;
apm_config.aec_delay_agnostic_enabled =
+ private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_delay_agnostic_enabled();
apm_config.aec_drift_compensation_enabled =
+ private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_drift_compensation_enabled();
apm_config.aec_extended_filter_enabled =
+ private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_extended_filter_enabled();
- apm_config.aec_suppression_level = static_cast<int>(
- private_submodules_->echo_cancellation->suppression_level());
+ apm_config.aec_suppression_level =
+ private_submodules_->echo_cancellation
+ ? static_cast<int>(
+ private_submodules_->echo_cancellation->suppression_level())
+ : 0;
apm_config.aecm_enabled =
private_submodules_->echo_control_mobile->is_enabled();
@@ -2115,7 +2145,9 @@
AecDump::AudioProcessingState audio_proc_state;
audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
audio_proc_state.drift =
- private_submodules_->echo_cancellation->stream_drift_samples();
+ private_submodules_->echo_cancellation
+ ? private_submodules_->echo_cancellation->stream_drift_samples()
+ : 0;
audio_proc_state.level = agc1()->stream_analog_level();
audio_proc_state.keypress = capture_.key_pressed;
aec_dump_->AddAudioProcessingState(audio_proc_state);
diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h
index 9cd4f78..33cc559 100644
--- a/modules/audio_processing/audio_processing_impl.h
+++ b/modules/audio_processing/audio_processing_impl.h
@@ -246,7 +246,8 @@
void InitializeResidualEchoDetector()
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
void InitializeLowCutFilter() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
- void InitializeEchoController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
+ void InitializeEchoController()
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
@@ -417,6 +418,9 @@
int split_rate;
int stream_delay_ms;
bool echo_controller_enabled = false;
+ bool use_aec2_extended_filter = false;
+ bool use_aec2_delay_agnostic = false;
+ bool use_aec2_refined_adaptive_filter = false;
} capture_nonlocked_;
struct ApmRenderState {
diff --git a/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc b/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc
index af0741e..37898c0 100644
--- a/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc
+++ b/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc
@@ -33,7 +33,7 @@
Config config;
config.Set<DelayAgnostic>(new DelayAgnostic(true));
config.Set<ExtendedFilter>(new ExtendedFilter(true));
- echo_canceller->SetExtraOptions(config);
+ echo_canceller->SetExtraOptions(true, true, false);
}
void ProcessOneFrame(int sample_rate_hz,
diff --git a/modules/audio_processing/echo_cancellation_impl.cc b/modules/audio_processing/echo_cancellation_impl.cc
index 96b9789..ac13ebe 100644
--- a/modules/audio_processing/echo_cancellation_impl.cc
+++ b/modules/audio_processing/echo_cancellation_impl.cc
@@ -16,7 +16,6 @@
#include "modules/audio_processing/aec/aec_core.h"
#include "modules/audio_processing/aec/echo_cancellation.h"
#include "modules/audio_processing/audio_buffer.h"
-#include "modules/audio_processing/include/config.h"
#include "rtc_base/checks.h"
#include "system_wrappers/include/field_trial.h"
@@ -432,13 +431,12 @@
}
}
-void EchoCancellationImpl::SetExtraOptions(const webrtc::Config& config) {
- {
- extended_filter_enabled_ = config.Get<ExtendedFilter>().enabled;
- delay_agnostic_enabled_ = config.Get<DelayAgnostic>().enabled;
- refined_adaptive_filter_enabled_ =
- config.Get<RefinedAdaptiveFilter>().enabled;
- }
+void EchoCancellationImpl::SetExtraOptions(bool use_extended_filter,
+ bool use_delay_agnostic,
+ bool use_refined_adaptive_filter) {
+ extended_filter_enabled_ = use_extended_filter;
+ delay_agnostic_enabled_ = use_delay_agnostic;
+ refined_adaptive_filter_enabled_ = use_refined_adaptive_filter;
Configure();
}
diff --git a/modules/audio_processing/echo_cancellation_impl.h b/modules/audio_processing/echo_cancellation_impl.h
index 79be73b..25412cd 100644
--- a/modules/audio_processing/echo_cancellation_impl.h
+++ b/modules/audio_processing/echo_cancellation_impl.h
@@ -17,10 +17,7 @@
#include <vector>
#include "api/array_view.h"
-#include "modules/audio_processing/include/audio_processing.h"
#include "rtc_base/constructor_magic.h"
-#include "rtc_base/critical_section.h"
-#include "rtc_base/thread_annotations.h"
namespace webrtc {
@@ -136,7 +133,9 @@
size_t num_reverse_channels_,
size_t num_output_channels_,
size_t num_proc_channels_);
- void SetExtraOptions(const webrtc::Config& config);
+ void SetExtraOptions(bool use_extended_filter,
+ bool use_delay_agnostic,
+ bool use_refined_adaptive_filter);
bool is_delay_agnostic_enabled() const;
bool is_extended_filter_enabled() const;
std::string GetExperimentsDescription();
diff --git a/modules/audio_processing/echo_cancellation_impl_unittest.cc b/modules/audio_processing/echo_cancellation_impl_unittest.cc
index 841ea0d..22741ee 100644
--- a/modules/audio_processing/echo_cancellation_impl_unittest.cc
+++ b/modules/audio_processing/echo_cancellation_impl_unittest.cc
@@ -31,16 +31,14 @@
EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(aec_core));
Config config;
- config.Set<ExtendedFilter>(new ExtendedFilter(true));
- echo_canceller.SetExtraOptions(config);
+ echo_canceller.SetExtraOptions(true, false, false);
EXPECT_EQ(1, WebRtcAec_extended_filter_enabled(aec_core));
// Retains setting after initialization.
echo_canceller.Initialize(AudioProcessing::kSampleRate16kHz, 2, 2, 2);
EXPECT_EQ(1, WebRtcAec_extended_filter_enabled(aec_core));
- config.Set<ExtendedFilter>(new ExtendedFilter(false));
- echo_canceller.SetExtraOptions(config);
+ echo_canceller.SetExtraOptions(false, false, false);
EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(aec_core));
// Retains setting after initialization.
@@ -63,8 +61,7 @@
EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(aec_core));
Config config;
- config.Set<DelayAgnostic>(new DelayAgnostic(true));
- echo_canceller.SetExtraOptions(config);
+ echo_canceller.SetExtraOptions(false, true, false);
EXPECT_EQ(1, WebRtcAec_delay_agnostic_enabled(aec_core));
// Retains setting after initialization.
@@ -72,7 +69,7 @@
EXPECT_EQ(1, WebRtcAec_delay_agnostic_enabled(aec_core));
config.Set<DelayAgnostic>(new DelayAgnostic(false));
- echo_canceller.SetExtraOptions(config);
+ echo_canceller.SetExtraOptions(false, false, false);
EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(aec_core));
// Retains setting after initialization.
