blob: 7b306ba3b937f8313e7d27c755a5a093faf36d1f [file] [log] [blame]
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/aec3/residual_echo_estimator.h"
#include <numeric>
#include <vector>
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
// Estimates the echo generating signal power as gated maximal power over a time
// window.
void EchoGeneratingPower(const RenderBuffer& render_buffer,
size_t min_delay,
size_t max_delay,
std::array<float, kFftLengthBy2Plus1>* X2) {
X2->fill(0.f);
for (size_t k = min_delay; k <= max_delay; ++k) {
std::transform(X2->begin(), X2->end(), render_buffer.Spectrum(k).begin(),
X2->begin(),
[](float a, float b) { return std::max(a, b); });
}
// Apply soft noise gate of -78 dBFS.
static constexpr float kNoiseGatePower = 27509.42f;
std::for_each(X2->begin(), X2->end(), [](float& a) {
if (kNoiseGatePower > a) {
a = std::max(0.f, a - 0.3f * (kNoiseGatePower - a));
}
});
}
constexpr int kNoiseFloorCounterMax = 50;
constexpr float kNoiseFloorMin = 10.f * 10.f * 128.f * 128.f;
// Updates estimate for the power of the stationary noise component in the
// render signal.
void RenderNoisePower(
const RenderBuffer& render_buffer,
std::array<float, kFftLengthBy2Plus1>* X2_noise_floor,
std::array<int, kFftLengthBy2Plus1>* X2_noise_floor_counter) {
RTC_DCHECK(X2_noise_floor);
RTC_DCHECK(X2_noise_floor_counter);
const auto render_power = render_buffer.Spectrum(0);
RTC_DCHECK_EQ(X2_noise_floor->size(), render_power.size());
RTC_DCHECK_EQ(X2_noise_floor_counter->size(), render_power.size());
// Estimate the stationary noise power in a minimum statistics manner.
for (size_t k = 0; k < render_power.size(); ++k) {
// Decrease rapidly.
if (render_power[k] < (*X2_noise_floor)[k]) {
(*X2_noise_floor)[k] = render_power[k];
(*X2_noise_floor_counter)[k] = 0;
} else {
// Increase in a delayed, leaky manner.
if ((*X2_noise_floor_counter)[k] >= kNoiseFloorCounterMax) {
(*X2_noise_floor)[k] =
std::max((*X2_noise_floor)[k] * 1.1f, kNoiseFloorMin);
} else {
++(*X2_noise_floor_counter)[k];
}
}
}
}
} // namespace
ResidualEchoEstimator::ResidualEchoEstimator(const EchoCanceller3Config& config)
: config_(config) {
Reset();
}
ResidualEchoEstimator::~ResidualEchoEstimator() = default;
void ResidualEchoEstimator::Estimate(
const AecState& aec_state,
const RenderBuffer& render_buffer,
const std::array<float, kFftLengthBy2Plus1>& S2_linear,
const std::array<float, kFftLengthBy2Plus1>& Y2,
std::array<float, kFftLengthBy2Plus1>* R2) {
RTC_DCHECK(R2);
// Estimate the power of the stationary noise in the render signal.
RenderNoisePower(render_buffer, &X2_noise_floor_, &X2_noise_floor_counter_);
// Estimate the residual echo power.
if (aec_state.LinearEchoEstimate()) {
RTC_DCHECK(aec_state.FilterDelay());
const int filter_delay = *aec_state.FilterDelay();
LinearEstimate(S2_linear, aec_state.Erle(), filter_delay, R2);
AddEchoReverb(S2_linear, aec_state.SaturatedEcho(), filter_delay,
aec_state.ReverbDecay(), R2);
// If the echo is saturated, estimate the echo power as the maximum echo
// power with a leakage factor.
if (aec_state.SaturatedEcho()) {
R2->fill((*std::max_element(R2->begin(), R2->end())) * 100.f);
}
} else {
const rtc::Optional<size_t> delay =
aec_state.ExternalDelay()
? (aec_state.FilterDelay() ? aec_state.FilterDelay()
: aec_state.ExternalDelay())
: rtc::Optional<size_t>();
// Estimate the echo generating signal power.
std::array<float, kFftLengthBy2Plus1> X2;
if (aec_state.ExternalDelay() && aec_state.FilterDelay()) {
RTC_DCHECK(delay);
const int delay_use = static_cast<int>(*delay);
// Computes the spectral power over the blocks surrounding the delay.
constexpr int kKnownDelayRenderWindowSize = 5;
static_assert(
kUnknownDelayRenderWindowSize >= kKnownDelayRenderWindowSize,
"Requirement to ensure that the render buffer is overrun");
EchoGeneratingPower(
render_buffer, std::max(0, delay_use - 1),
std::min(kKnownDelayRenderWindowSize - 1, delay_use + 1), &X2);
} else {
// Computes the spectral power over the latest blocks.
EchoGeneratingPower(render_buffer, 0, kUnknownDelayRenderWindowSize - 1,
&X2);
}
// Subtract the stationary noise power to avoid stationary noise causing
// excessive echo suppression.
std::transform(
X2.begin(), X2.end(), X2_noise_floor_.begin(), X2.begin(),
[](float a, float b) { return std::max(0.f, a - 10.f * b); });
NonLinearEstimate(
aec_state.SufficientFilterUpdates(), aec_state.SaturatedEcho(),
config_.ep_strength.bounded_erl, aec_state.TransparentMode(),
aec_state.InitialState(), X2, Y2, R2);
if (aec_state.ExternalDelay() && aec_state.FilterDelay() &&
aec_state.SaturatedEcho()) {
AddEchoReverb(*R2, aec_state.SaturatedEcho(),
std::min(static_cast<size_t>(kAdaptiveFilterLength),
delay.value_or(kAdaptiveFilterLength)),
aec_state.ReverbDecay(), R2);
}
}
// If the echo is deemed inaudible, set the residual echo to zero.
if (aec_state.InaudibleEcho()) {
R2->fill(0.f);
R2_old_.fill(0.f);
R2_hold_counter_.fill(0.f);
}
std::copy(R2->begin(), R2->end(), R2_old_.begin());
}
void ResidualEchoEstimator::Reset() {
X2_noise_floor_counter_.fill(kNoiseFloorCounterMax);
X2_noise_floor_.fill(kNoiseFloorMin);
R2_reverb_.fill(0.f);
R2_old_.fill(0.f);
R2_hold_counter_.fill(0.f);
for (auto& S2_k : S2_old_) {
S2_k.fill(0.f);
}
}
void ResidualEchoEstimator::LinearEstimate(
const std::array<float, kFftLengthBy2Plus1>& S2_linear,
const std::array<float, kFftLengthBy2Plus1>& erle,
size_t delay,
std::array<float, kFftLengthBy2Plus1>* R2) {
std::fill(R2_hold_counter_.begin(), R2_hold_counter_.end(), 10.f);
std::transform(erle.begin(), erle.end(), S2_linear.begin(), R2->begin(),
[](float a, float b) {
RTC_DCHECK_LT(0.f, a);
return b / a;
});
}
void ResidualEchoEstimator::NonLinearEstimate(
bool sufficient_filter_updates,
bool saturated_echo,
bool bounded_erl,
bool transparent_mode,
bool initial_state,
const std::array<float, kFftLengthBy2Plus1>& X2,
const std::array<float, kFftLengthBy2Plus1>& Y2,
std::array<float, kFftLengthBy2Plus1>* R2) {
float echo_path_gain_lf;
float echo_path_gain_mf;
float echo_path_gain_hf;
// Set echo path gains.
if (saturated_echo) {
// If the echo could be saturated, use a very conservative gain.
echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 10000.f;
} else if (sufficient_filter_updates && !bounded_erl) {
// If the filter should have been able to converge, and no assumption is
// possible on the ERL, use a low gain.
echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 0.01f;
} else if ((sufficient_filter_updates && bounded_erl) || transparent_mode) {
// If the filter should have been able to converge, and and it is known that
// the ERL is bounded, use a very low gain.
echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 0.001f;
} else if (!initial_state) {
// If the AEC is no longer in an initial state, assume a weak echo path.
echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 0.01f;
} else {
// In the initial state, use conservative gains.
echo_path_gain_lf = config_.ep_strength.lf;
echo_path_gain_mf = config_.ep_strength.mf;
echo_path_gain_hf = config_.ep_strength.hf;
}
// Compute preliminary residual echo.
std::transform(
X2.begin(), X2.begin() + 12, R2->begin(),
[echo_path_gain_lf](float a) { return a * echo_path_gain_lf; });
std::transform(
X2.begin() + 12, X2.begin() + 25, R2->begin() + 12,
[echo_path_gain_mf](float a) { return a * echo_path_gain_mf; });
std::transform(
X2.begin() + 25, X2.end(), R2->begin() + 25,
[echo_path_gain_hf](float a) { return a * echo_path_gain_hf; });
for (size_t k = 0; k < R2->size(); ++k) {
// Update hold counter.
R2_hold_counter_[k] = R2_old_[k] < (*R2)[k] ? 0 : R2_hold_counter_[k] + 1;
// Compute the residual echo by holding a maximum echo powers and an echo
// fading corresponding to a room with an RT60 value of about 50 ms.
(*R2)[k] = R2_hold_counter_[k] < 2
? std::max((*R2)[k], R2_old_[k])
: std::min((*R2)[k] + R2_old_[k] * 0.1f, Y2[k]);
}
}
void ResidualEchoEstimator::AddEchoReverb(
const std::array<float, kFftLengthBy2Plus1>& S2,
bool saturated_echo,
size_t delay,
float reverb_decay_factor,
std::array<float, kFftLengthBy2Plus1>* R2) {
// Compute the decay factor for how much the echo has decayed before leaving
// the region covered by the linear model.
auto integer_power = [](float base, int exp) {
float result = 1.f;
for (int k = 0; k < exp; ++k) {
result *= base;
}
return result;
};
RTC_DCHECK_LE(delay, S2_old_.size());
const float reverb_decay_for_delay =
integer_power(reverb_decay_factor, S2_old_.size() - delay);
// Update the estimate of the reverberant residual echo power.
S2_old_index_ = S2_old_index_ > 0 ? S2_old_index_ - 1 : S2_old_.size() - 1;
const auto& S2_end = S2_old_[S2_old_index_];
std::transform(
S2_end.begin(), S2_end.end(), R2_reverb_.begin(), R2_reverb_.begin(),
[reverb_decay_for_delay, reverb_decay_factor](float a, float b) {
return (b + a * reverb_decay_for_delay) * reverb_decay_factor;
});
// Update the buffer of old echo powers.
if (saturated_echo) {
S2_old_[S2_old_index_].fill((*std::max_element(S2.begin(), S2.end())) *
100.f);
} else {
std::copy(S2.begin(), S2.end(), S2_old_[S2_old_index_].begin());
}
// Add the power of the echo reverb to the residual echo power.
std::transform(R2->begin(), R2->end(), R2_reverb_.begin(), R2->begin(),
std::plus<float>());
}
} // namespace webrtc