| /* |
| * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| #include "modules/audio_processing/aec3/block_processor.h" |
| |
| #include "api/optional.h" |
| #include "modules/audio_processing/aec3/aec3_common.h" |
| #include "modules/audio_processing/aec3/block_processor_metrics.h" |
| #include "modules/audio_processing/aec3/echo_path_variability.h" |
| #include "modules/audio_processing/logging/apm_data_dumper.h" |
| #include "rtc_base/atomicops.h" |
| #include "rtc_base/constructormagic.h" |
| #include "rtc_base/logging.h" |
| |
| namespace webrtc { |
| namespace { |
| |
| enum class BlockProcessorApiCall { kCapture, kRender }; |
| |
| class BlockProcessorImpl final : public BlockProcessor { |
| public: |
| BlockProcessorImpl(const EchoCanceller3Config& config, |
| int sample_rate_hz, |
| std::unique_ptr<RenderDelayBuffer> render_buffer, |
| std::unique_ptr<RenderDelayController> delay_controller, |
| std::unique_ptr<EchoRemover> echo_remover); |
| |
| ~BlockProcessorImpl() override; |
| |
| void ProcessCapture(bool echo_path_gain_change, |
| bool capture_signal_saturation, |
| std::vector<std::vector<float>>* capture_block) override; |
| |
| void BufferRender(const std::vector<std::vector<float>>& block) override; |
| |
| void UpdateEchoLeakageStatus(bool leakage_detected) override; |
| |
| void GetMetrics(EchoControl::Metrics* metrics) const override; |
| |
| private: |
| static int instance_count_; |
| std::unique_ptr<ApmDataDumper> data_dumper_; |
| const EchoCanceller3Config config_; |
| bool capture_properly_started_ = false; |
| bool render_properly_started_ = false; |
| const size_t sample_rate_hz_; |
| std::unique_ptr<RenderDelayBuffer> render_buffer_; |
| std::unique_ptr<RenderDelayController> delay_controller_; |
| std::unique_ptr<EchoRemover> echo_remover_; |
| BlockProcessorMetrics metrics_; |
| RenderDelayBuffer::BufferingEvent render_event_; |
| RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl); |
| }; |
| |
| int BlockProcessorImpl::instance_count_ = 0; |
| |
| BlockProcessorImpl::BlockProcessorImpl( |
| const EchoCanceller3Config& config, |
| int sample_rate_hz, |
| std::unique_ptr<RenderDelayBuffer> render_buffer, |
| std::unique_ptr<RenderDelayController> delay_controller, |
| std::unique_ptr<EchoRemover> echo_remover) |
| : data_dumper_( |
| new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), |
| config_(config), |
| sample_rate_hz_(sample_rate_hz), |
| render_buffer_(std::move(render_buffer)), |
| delay_controller_(std::move(delay_controller)), |
| echo_remover_(std::move(echo_remover)), |
| render_event_(RenderDelayBuffer::BufferingEvent::kNone) { |
| RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); |
| } |
| |
| BlockProcessorImpl::~BlockProcessorImpl() = default; |
| |
| void BlockProcessorImpl::ProcessCapture( |
| bool echo_path_gain_change, |
| bool capture_signal_saturation, |
| std::vector<std::vector<float>>* capture_block) { |
| RTC_DCHECK(capture_block); |
| RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size()); |
| RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size()); |
| data_dumper_->DumpRaw("aec3_processblock_call_order", |
| static_cast<int>(BlockProcessorApiCall::kCapture)); |
| data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize, |
| &(*capture_block)[0][0], |
| LowestBandRate(sample_rate_hz_), 1); |
| |
| if (render_properly_started_) { |
| if (!capture_properly_started_) { |
| capture_properly_started_ = true; |
| render_buffer_->Reset(); |
| } |
| } else { |
| // If no render data has yet arrived, do not process the capture signal. |
| return; |
| } |
| |
| EchoPathVariability echo_path_variability( |
| echo_path_gain_change, EchoPathVariability::DelayAdjustment::kNone, |
| false); |
| |
| if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderOverrun && |
| render_properly_started_) { |
| echo_path_variability.delay_change = |
| EchoPathVariability::DelayAdjustment::kBufferFlush; |
| delay_controller_->Reset(); |
| render_buffer_->Reset(); |
| RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun."; |
| } |
| |
| // Update the render buffers with any newly arrived render blocks and prepare |
| // the render buffers for reading the render data corresponding to the current |
| // capture block. |
| render_event_ = render_buffer_->PrepareCaptureCall(); |
| RTC_DCHECK(RenderDelayBuffer::BufferingEvent::kRenderOverrun != |
| render_event_); |
| if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) { |
| echo_path_variability.delay_change = |
| EchoPathVariability::DelayAdjustment::kBufferReadjustment; |
| delay_controller_->Reset(); |
| render_buffer_->Reset(); |
| } else if (render_event_ == RenderDelayBuffer::BufferingEvent::kApiCallSkew) { |
| // There have been too many render calls in a row. Reset to avoid noncausal |
| // echo. |
| echo_path_variability.delay_change = |
| EchoPathVariability::DelayAdjustment::kDelayReset; |
| delay_controller_->Reset(); |
| render_buffer_->Reset(); |
| capture_properly_started_ = false; |
| render_properly_started_ = false; |
| } |
| |
| data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, |
| &(*capture_block)[0][0], |
| LowestBandRate(sample_rate_hz_), 1); |
| |
| // Compute and and apply the render delay required to achieve proper signal |
| // alignment. |
| const size_t estimated_delay = delay_controller_->GetDelay( |
| render_buffer_->GetDownsampledRenderBuffer(), (*capture_block)[0]); |
| const size_t new_delay = |
| std::min(render_buffer_->MaxDelay(), estimated_delay); |
| |
| bool delay_change = render_buffer_->Delay() != new_delay; |
| if (delay_change && new_delay >= config_.delay.min_echo_path_delay_blocks) { |
| echo_path_variability.delay_change = |
| EchoPathVariability::DelayAdjustment::kNewDetectedDelay; |
| render_buffer_->SetDelay(new_delay); |
| RTC_DCHECK_EQ(render_buffer_->Delay(), new_delay); |
| delay_controller_->SetDelay(new_delay); |
| } else if (delay_change && |
| new_delay < config_.delay.min_echo_path_delay_blocks) { |
| // A noncausal delay has been detected. This can only happen if there is |
| // clockdrift, an audio pipeline issue has occurred or the specified minimum |
| // delay is too short. Perform a full reset. |
| echo_path_variability.delay_change = |
| EchoPathVariability::DelayAdjustment::kDelayReset; |
| delay_controller_->Reset(); |
| render_buffer_->Reset(); |
| capture_properly_started_ = false; |
| render_properly_started_ = false; |
| RTC_LOG(LS_WARNING) << "Reset due to noncausal delay."; |
| } |
| |
| // Remove the echo from the capture signal. |
| echo_remover_->ProcessCapture( |
| delay_controller_->AlignmentHeadroomSamples(), echo_path_variability, |
| capture_signal_saturation, render_buffer_->GetRenderBuffer(), |
| capture_block); |
| |
| // Update the metrics. |
| metrics_.UpdateCapture(false); |
| |
| render_event_ = RenderDelayBuffer::BufferingEvent::kNone; |
| } |
| |
| void BlockProcessorImpl::BufferRender( |
| const std::vector<std::vector<float>>& block) { |
| RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size()); |
| RTC_DCHECK_EQ(kBlockSize, block[0].size()); |
| data_dumper_->DumpRaw("aec3_processblock_call_order", |
| static_cast<int>(BlockProcessorApiCall::kRender)); |
| data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize, |
| &block[0][0], LowestBandRate(sample_rate_hz_), 1); |
| data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize, |
| &block[0][0], LowestBandRate(sample_rate_hz_), 1); |
| |
| render_event_ = render_buffer_->Insert(block); |
| |
| metrics_.UpdateRender(render_event_ != |
| RenderDelayBuffer::BufferingEvent::kNone); |
| |
| render_properly_started_ = true; |
| } |
| |
| void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { |
| echo_remover_->UpdateEchoLeakageStatus(leakage_detected); |
| } |
| |
| void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const { |
| echo_remover_->GetMetrics(metrics); |
| const int block_size_ms = sample_rate_hz_ == 8000 ? 8 : 4; |
| metrics->delay_ms = static_cast<int>(render_buffer_->Delay()) * block_size_ms; |
| } |
| |
| } // namespace |
| |
| BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, |
| int sample_rate_hz) { |
| std::unique_ptr<RenderDelayBuffer> render_buffer( |
| RenderDelayBuffer::Create(config, NumBandsForRate(sample_rate_hz))); |
| std::unique_ptr<RenderDelayController> delay_controller( |
| RenderDelayController::Create(config, sample_rate_hz)); |
| std::unique_ptr<EchoRemover> echo_remover( |
| EchoRemover::Create(config, sample_rate_hz)); |
| return Create(config, sample_rate_hz, std::move(render_buffer), |
| std::move(delay_controller), std::move(echo_remover)); |
| } |
| |
| BlockProcessor* BlockProcessor::Create( |
| const EchoCanceller3Config& config, |
| int sample_rate_hz, |
| std::unique_ptr<RenderDelayBuffer> render_buffer) { |
| std::unique_ptr<RenderDelayController> delay_controller( |
| RenderDelayController::Create(config, sample_rate_hz)); |
| std::unique_ptr<EchoRemover> echo_remover( |
| EchoRemover::Create(config, sample_rate_hz)); |
| return Create(config, sample_rate_hz, std::move(render_buffer), |
| std::move(delay_controller), std::move(echo_remover)); |
| } |
| |
| BlockProcessor* BlockProcessor::Create( |
| const EchoCanceller3Config& config, |
| int sample_rate_hz, |
| std::unique_ptr<RenderDelayBuffer> render_buffer, |
| std::unique_ptr<RenderDelayController> delay_controller, |
| std::unique_ptr<EchoRemover> echo_remover) { |
| return new BlockProcessorImpl( |
| config, sample_rate_hz, std::move(render_buffer), |
| std::move(delay_controller), std::move(echo_remover)); |
| } |
| |
| } // namespace webrtc |