Leveraging the skew in API call order to a boost AEC3 signal realignment
This CL resets the AEC3 realignment functionality when a significant
and persistent skew in the number of render and capture API calls is
detected.
Bug: chromium:811658,webrtc:8879
Change-Id: Ib5c727b38f427da2a7d25eac7c939a17bdaabe74
Reviewed-on: https://webrtc-review.googlesource.com/52260
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Gustaf Ullberg <gustaf@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21997}diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc
index 24f2c91..dafe384 100644
--- a/modules/audio_processing/aec3/block_processor.cc
+++ b/modules/audio_processing/aec3/block_processor.cc
@@ -101,6 +101,7 @@
if (!capture_properly_started_) {
capture_properly_started_ = true;
render_buffer_->Reset();
+ delay_controller_->Reset();
}
} else {
// If no render data has yet arrived, do not process the capture signal.
@@ -213,6 +214,7 @@
RenderDelayBuffer::BufferingEvent::kNone);
render_properly_started_ = true;
+ delay_controller_->LogRenderCall();
}
void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) {
diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/modules/audio_processing/aec3/echo_path_delay_estimator.cc
index 95796d5..d9d753c 100644
--- a/modules/audio_processing/aec3/echo_path_delay_estimator.cc
+++ b/modules/audio_processing/aec3/echo_path_delay_estimator.cc
@@ -43,8 +43,10 @@
EchoPathDelayEstimator::~EchoPathDelayEstimator() = default;
-void EchoPathDelayEstimator::Reset() {
- matched_filter_lag_aggregator_.Reset();
+void EchoPathDelayEstimator::Reset(bool soft_reset) {
+ if (!soft_reset) {
+ matched_filter_lag_aggregator_.Reset();
+ }
matched_filter_.Reset();
}
diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.h b/modules/audio_processing/aec3/echo_path_delay_estimator.h
index ef0d448..10fa862 100644
--- a/modules/audio_processing/aec3/echo_path_delay_estimator.h
+++ b/modules/audio_processing/aec3/echo_path_delay_estimator.h
@@ -33,8 +33,9 @@
const EchoCanceller3Config& config);
~EchoPathDelayEstimator();
- // Resets the estimation.
- void Reset();
+ // Resets the estimation. If the soft-reset is specified, only the matched
+ // filters are reset.
+ void Reset(bool soft_reset);
// Produce a delay estimate if such is avaliable.
rtc::Optional<DelayEstimate> EstimateDelay(
diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h
index 4a82ae3..8fb7a8e 100644
--- a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h
+++ b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h
@@ -25,6 +25,7 @@
virtual ~MockRenderDelayController() = default;
MOCK_METHOD0(Reset, void());
+ MOCK_METHOD0(LogRenderCall, void());
MOCK_METHOD2(
GetDelay,
rtc::Optional<DelayEstimate>(const DownsampledRenderBuffer& render_buffer,
diff --git a/modules/audio_processing/aec3/render_delay_controller.cc b/modules/audio_processing/aec3/render_delay_controller.cc
index 3bc7d62..614f0ad 100644
--- a/modules/audio_processing/aec3/render_delay_controller.cc
+++ b/modules/audio_processing/aec3/render_delay_controller.cc
@@ -11,6 +11,7 @@
#include <algorithm>
#include <memory>
+#include <numeric>
#include <string>
#include <vector>
@@ -25,6 +26,47 @@
namespace {
+constexpr int kSkewHistorySizeLog2 = 8;
+
+// Estimator of API call skew between render and capture.
+class SkewEstimator {
+ public:
+ // Resets the estimation.
+ void Reset() {
+ skew_ = 0;
+ next_index_ = 0;
+ sufficient_skew_stored_ = false;
+ }
+
+ // Updates the skew data for a render call.
+ void LogRenderCall() { ++skew_; }
+
+ // Updates and computes the skew at a capture call. Returns an optional which
+ // is non-null if a reliable skew has been found.
+ rtc::Optional<int> GetSkewFromCapture() {
+ --skew_;
+
+ skew_history_[next_index_] = skew_;
+ if (++next_index_ == skew_history_.size()) {
+ next_index_ = 0;
+ sufficient_skew_stored_ = true;
+ }
+
+ if (!sufficient_skew_stored_) {
+ return rtc::nullopt;
+ }
+
+ return std::accumulate(skew_history_.begin(), skew_history_.end(), 0) >>
+ kSkewHistorySizeLog2;
+ }
+
+ private:
+ int skew_ = 0;
+ std::array<int, 1 << kSkewHistorySizeLog2> skew_history_;
+ size_t next_index_ = 0;
+ bool sufficient_skew_stored_ = false;
+};
+
class RenderDelayControllerImpl final : public RenderDelayController {
public:
RenderDelayControllerImpl(const EchoCanceller3Config& config,
@@ -32,6 +74,7 @@
int sample_rate_hz);
~RenderDelayControllerImpl() override;
void Reset() override;
+ void LogRenderCall() override;
rtc::Optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
rtc::ArrayView<const float> capture) override;
@@ -47,23 +90,27 @@
std::vector<float> delay_buf_;
int delay_buf_index_ = 0;
RenderDelayControllerMetrics metrics_;
+ SkewEstimator skew_estimator_;
+ rtc::Optional<DelayEstimate> delay_samples_;
+ rtc::Optional<int> skew_;
+ int delay_change_counter_ = 0;
+ size_t soft_reset_counter_ = 0;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl);
};
-DelayEstimate ComputeNewBufferDelay(
+DelayEstimate ComputeBufferDelay(
const rtc::Optional<DelayEstimate>& current_delay,
int delay_headroom_blocks,
int hysteresis_limit_1_blocks,
int hysteresis_limit_2_blocks,
+ int offset_blocks,
DelayEstimate estimated_delay) {
// The below division is not exact and the truncation is intended.
const int echo_path_delay_blocks = estimated_delay.delay >> kBlockSizeLog2;
// Compute the buffer delay increase required to achieve the desired latency.
- size_t new_delay_blocks =
- std::max(echo_path_delay_blocks - delay_headroom_blocks, 0);
-
- DelayEstimate new_delay(estimated_delay.quality, new_delay_blocks);
+ size_t new_delay_blocks = std::max(
+ echo_path_delay_blocks + offset_blocks - delay_headroom_blocks, 0);
// Add hysteresis.
if (current_delay) {
@@ -111,8 +158,17 @@
void RenderDelayControllerImpl::Reset() {
delay_ = rtc::nullopt;
+ delay_samples_ = rtc::nullopt;
+ skew_ = rtc::nullopt;
std::fill(delay_buf_.begin(), delay_buf_.end(), 0.f);
- delay_estimator_.Reset();
+ delay_estimator_.Reset(false);
+ skew_estimator_.Reset();
+ delay_change_counter_ = 0;
+ soft_reset_counter_ = 0;
+}
+
+void RenderDelayControllerImpl::LogRenderCall() {
+ skew_estimator_.LogRenderCall();
}
rtc::Optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
@@ -131,24 +187,64 @@
delay_buf_.begin() + delay_buf_index_);
delay_buf_index_ = (delay_buf_index_ + kBlockSize) % delay_buf_.size();
+ // Compute the latest skew update.
+ rtc::Optional<int> skew = skew_estimator_.GetSkewFromCapture();
+
if (delay_samples) {
- // Compute and set new render delay buffer delay.
-
- delay_ = ComputeNewBufferDelay(delay_, delay_headroom_blocks_,
- hysteresis_limit_1_blocks_,
- hysteresis_limit_2_blocks_, *delay_samples);
-
- metrics_.Update(static_cast<int>(delay_samples->delay),
- delay_ ? delay_->delay : 0);
- } else {
- metrics_.Update(rtc::nullopt, delay_ ? delay_->delay : 0);
+ if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
+ delay_change_counter_ = 0;
+ }
+ delay_samples_ = delay_samples;
}
+ if (delay_change_counter_ < 2 * kNumBlocksPerSecond) {
+ ++delay_change_counter_;
+ // If a new delay estimate is recently obtained, store the skew for that.
+ skew_ = skew;
+ } else {
+ // A reliable skew should have been obtained after 2 seconds.
+ RTC_DCHECK(skew_);
+ RTC_DCHECK(skew);
+ }
+
+ ++soft_reset_counter_;
+ int offset_blocks = 0;
+ if (skew_ && skew && delay_samples_ &&
+ delay_samples_->quality == DelayEstimate::Quality::kRefined) {
+ // Compute the skew offset and add a margin.
+ offset_blocks = *skew_ - *skew;
+ if (abs(offset_blocks) <= 1) {
+ offset_blocks = 0;
+ } else if (soft_reset_counter_ > 10 * kNumBlocksPerSecond) {
+ // Soft reset the delay estimator if there is a significant offset
+ // detected.
+ delay_estimator_.Reset(true);
+ soft_reset_counter_ = 0;
+ }
+ }
+
+ if (delay_samples_) {
+ // Compute the render delay buffer delay.
+ delay_ = ComputeBufferDelay(
+ delay_, delay_headroom_blocks_, hysteresis_limit_1_blocks_,
+ hysteresis_limit_2_blocks_, offset_blocks, *delay_samples_);
+ }
+
+ metrics_.Update(delay_samples_ ? rtc::Optional<size_t>(delay_samples_->delay)
+ : rtc::nullopt,
+ delay_ ? delay_->delay : 0);
+
data_dumper_->DumpRaw("aec3_render_delay_controller_delay",
delay_samples ? delay_samples->delay : 0);
data_dumper_->DumpRaw("aec3_render_delay_controller_buffer_delay",
delay_ ? delay_->delay : 0);
+ data_dumper_->DumpRaw("aec3_render_delay_controller_new_skew",
+ skew ? *skew : 0);
+ data_dumper_->DumpRaw("aec3_render_delay_controller_old_skew",
+ skew_ ? *skew_ : 0);
+ data_dumper_->DumpRaw("aec3_render_delay_controller_offset", offset_blocks);
+
return delay_;
}
diff --git a/modules/audio_processing/aec3/render_delay_controller.h b/modules/audio_processing/aec3/render_delay_controller.h
index 5b1fc35..842b8ff 100644
--- a/modules/audio_processing/aec3/render_delay_controller.h
+++ b/modules/audio_processing/aec3/render_delay_controller.h
@@ -32,6 +32,9 @@
// Resets the delay controller.
virtual void Reset() = 0;
+ // Logs a render call.
+ virtual void LogRenderCall() = 0;
+
// Aligns the render buffer content with the capture signal.
virtual rtc::Optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
