[AudioProcessingImpl] Don't use AudioFrameView with aec dump In AudioProcessingImpl we receive externally provided channel data that may not be byte aligned (e.g AudioBus in chromium is aligned on a 16 byte boundary). Bug: chromium:409777951 Change-Id: I7ad9afda6f4bbfa67cfaf2ea89a40bebf4624e8b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/392322 Commit-Queue: Tomas Gunnarsson <tommi@webrtc.org> Reviewed-by: Per Åhgren <peah@webrtc.org> Cr-Commit-Position: refs/heads/main@{#44717}
diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn index b42ded6..d36c3c4 100644 --- a/modules/audio_processing/BUILD.gn +++ b/modules/audio_processing/BUILD.gn
@@ -91,6 +91,7 @@ deps = [ ":audio_frame_view", + "../../api/audio:audio_frame_api", "../../api/audio:audio_processing", "//third_party/abseil-cpp/absl/base:core_headers", ] @@ -156,6 +157,7 @@ "../../api:field_trials_view", "../../api:function_view", "../../api:make_ref_counted", + "../../api:scoped_refptr", "../../api/audio:aec3_config", "../../api/audio:audio_frame_api", "../../api/audio:audio_processing", @@ -189,6 +191,7 @@ "agc", "agc:gain_control_interface", "agc:legacy_agc", + "agc2:input_volume_controller", "agc2:input_volume_stats_reporter", "capture_levels_adjuster", "ns",
diff --git a/modules/audio_processing/aec_dump/BUILD.gn b/modules/audio_processing/aec_dump/BUILD.gn index 5fb2942..b747aaa 100644 --- a/modules/audio_processing/aec_dump/BUILD.gn +++ b/modules/audio_processing/aec_dump/BUILD.gn
@@ -34,6 +34,9 @@ "..:aec_dump_interface", "..:audioproc_test_utils", "../", + "../../../api/audio:audio_frame_api", + "../../../api/audio:audio_processing", + "../../../modules/audio_processing:audio_frame_view", "../../../test:test_support", ] } @@ -47,9 +50,12 @@ ":mock_aec_dump", "..:audioproc_test_utils", "../", + "../../../api:scoped_refptr", "../../../api/audio:audio_processing", "../../../api/audio:builtin_audio_processing_builder", "../../../api/environment:environment_factory", + "../../../rtc_base:checks", + "../../../test:test_support", "//testing/gtest", ] } @@ -68,7 +74,9 @@ ":aec_dump", "..:aec_dump_interface", "../../../api/audio:audio_frame_api", + "../../../api/audio:audio_processing", "../../../api/task_queue", + "../../../modules/audio_processing:audio_frame_view", "../../../rtc_base:checks", "../../../rtc_base:logging", "../../../rtc_base:macromagic",
diff --git a/modules/audio_processing/aec_dump/aec_dump_impl.cc b/modules/audio_processing/aec_dump/aec_dump_impl.cc index 31826ff..92f3bd3 100644 --- a/modules/audio_processing/aec_dump/aec_dump_impl.cc +++ b/modules/audio_processing/aec_dump/aec_dump_impl.cc
@@ -10,15 +10,26 @@ #include "modules/audio_processing/aec_dump/aec_dump_impl.h" +#include <cstddef> +#include <cstdint> +#include <cstdio> #include <memory> +#include <string> #include <utility> #include "absl/base/nullability.h" #include "absl/strings/string_view.h" +#include "api/audio/audio_processing.h" +#include "api/audio/audio_view.h" #include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "modules/audio_processing/debug.pb.h" +#include "modules/audio_processing/include/aec_dump.h" +#include "modules/audio_processing/include/audio_frame_view.h" #include "rtc_base/checks.h" #include "rtc_base/event.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/system/file_wrapper.h" namespace webrtc { @@ -105,11 +116,19 @@ capture_stream_info_.AddInput(src); } +void AecDumpImpl::AddCaptureStreamInput(MonoView<const float> channel) { + capture_stream_info_.AddInputChannel(channel); +} + void AecDumpImpl::AddCaptureStreamOutput( const AudioFrameView<const float>& src) { capture_stream_info_.AddOutput(src); } +void AecDumpImpl::AddCaptureStreamOutput(MonoView<const float> channel) { + capture_stream_info_.AddOutputChannel(channel); +} + void AecDumpImpl::AddCaptureStreamInput(const int16_t* const data, int num_channels, int samples_per_channel) { @@ -157,6 +176,22 @@ PostWriteToFileTask(std::move(event)); } +void AecDumpImpl::WriteRenderStreamMessage(const float* const* data, + int num_channels, + int samples_per_channel) { + auto event = std::make_unique<audioproc::Event>(); + event->set_type(audioproc::Event::REVERSE_STREAM); + + audioproc::ReverseStream* msg = event->mutable_reverse_stream(); + + for (int i = 0; i < num_channels; ++i) { + MonoView<const float> channel_view(data[i], samples_per_channel); + msg->add_channel(channel_view.begin(), sizeof(float) * channel_view.size()); + } + + PostWriteToFileTask(std::move(event)); +} + void AecDumpImpl::WriteConfig(const InternalAPMConfig& config) { RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); auto event = std::make_unique<audioproc::Event>();
diff --git a/modules/audio_processing/aec_dump/aec_dump_impl.h b/modules/audio_processing/aec_dump/aec_dump_impl.h index 7638758..aa25eae 100644 --- a/modules/audio_processing/aec_dump/aec_dump_impl.h +++ b/modules/audio_processing/aec_dump/aec_dump_impl.h
@@ -11,16 +11,18 @@ #ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_IMPL_H_ #define MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_IMPL_H_ +#include <cstdint> #include <memory> -#include <string> -#include <vector> +#include "absl/base/nullability.h" +#include "api/audio/audio_processing.h" +#include "api/audio/audio_view.h" #include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/aec_dump/capture_stream_info.h" #include "modules/audio_processing/include/aec_dump.h" +#include "modules/audio_processing/include/audio_frame_view.h" #include "rtc_base/race_checker.h" #include "rtc_base/system/file_wrapper.h" -#include "rtc_base/thread_annotations.h" // Files generated at build-time by the protobuf compiler. #ifdef WEBRTC_ANDROID_PLATFORM_BUILD @@ -47,7 +49,9 @@ void WriteInitMessage(const ProcessingConfig& api_format, int64_t time_now_ms) override; void AddCaptureStreamInput(const AudioFrameView<const float>& src) override; + void AddCaptureStreamInput(MonoView<const float> channel) override; void AddCaptureStreamOutput(const AudioFrameView<const float>& src) override; + void AddCaptureStreamOutput(MonoView<const float> channel) override; void AddCaptureStreamInput(const int16_t* const data, int num_channels, int samples_per_channel) override; @@ -62,6 +66,9 @@ int samples_per_channel) override; void WriteRenderStreamMessage( const AudioFrameView<const float>& src) override; + void WriteRenderStreamMessage(const float* const* data, + int num_channels, + int samples_per_channel) override; void WriteConfig(const InternalAPMConfig& config) override;
diff --git a/modules/audio_processing/aec_dump/aec_dump_integration_test.cc b/modules/audio_processing/aec_dump/aec_dump_integration_test.cc index 5c23c96..7701877 100644 --- a/modules/audio_processing/aec_dump/aec_dump_integration_test.cc +++ b/modules/audio_processing/aec_dump/aec_dump_integration_test.cc
@@ -9,18 +9,23 @@ */ #include <array> +#include <cstdint> #include <memory> #include <utility> #include "api/audio/audio_processing.h" #include "api/audio/builtin_audio_processing_builder.h" #include "api/environment/environment_factory.h" +#include "api/scoped_refptr.h" #include "modules/audio_processing/aec_dump/mock_aec_dump.h" -#include "modules/audio_processing/audio_processing_impl.h" +#include "rtc_base/checks.h" +#include "test/gmock.h" +#include "test/gtest.h" using ::testing::_; using ::testing::AtLeast; using ::testing::Exactly; +using ::testing::Matcher; using ::testing::StrictMock; namespace { @@ -59,7 +64,8 @@ frame.fill(0.f); webrtc::StreamConfig stream_config(kNumSampleRateHz, kNumChannels); - EXPECT_CALL(*mock_aec_dump.get(), WriteRenderStreamMessage(_, _, _)) + EXPECT_CALL(*mock_aec_dump.get(), + WriteRenderStreamMessage(Matcher<const int16_t*>(_), _, _)) .Times(Exactly(1)); apm->AttachAecDump(std::move(mock_aec_dump));
diff --git a/modules/audio_processing/aec_dump/capture_stream_info.cc b/modules/audio_processing/aec_dump/capture_stream_info.cc index 7d82a39..9933866 100644 --- a/modules/audio_processing/aec_dump/capture_stream_info.cc +++ b/modules/audio_processing/aec_dump/capture_stream_info.cc
@@ -10,28 +10,37 @@ #include "modules/audio_processing/aec_dump/capture_stream_info.h" +#include <cstddef> +#include <cstdint> + +#include "api/audio/audio_view.h" +#include "modules/audio_processing/include/aec_dump.h" +#include "modules/audio_processing/include/audio_frame_view.h" + namespace webrtc { void CaptureStreamInfo::AddInput(const AudioFrameView<const float>& src) { - auto* stream = event_->mutable_stream(); - for (int i = 0; i < src.num_channels(); ++i) { - const auto& channel_view = src.channel(i); - stream->add_input_channel(channel_view.begin(), - sizeof(float) * channel_view.size()); + AddInputChannel(src.channel(i)); } } -void CaptureStreamInfo::AddOutput(const AudioFrameView<const float>& src) { +void CaptureStreamInfo::AddInputChannel(MonoView<const float> channel) { auto* stream = event_->mutable_stream(); + stream->add_input_channel(channel.begin(), sizeof(float) * channel.size()); +} +void CaptureStreamInfo::AddOutput(const AudioFrameView<const float>& src) { for (int i = 0; i < src.num_channels(); ++i) { - const auto& channel_view = src.channel(i); - stream->add_output_channel(channel_view.begin(), - sizeof(float) * channel_view.size()); + AddOutputChannel(src.channel(i)); } } +void CaptureStreamInfo::AddOutputChannel(MonoView<const float> channel) { + auto* stream = event_->mutable_stream(); + stream->add_output_channel(channel.begin(), sizeof(float) * channel.size()); +} + void CaptureStreamInfo::AddInput(const int16_t* const data, int num_channels, int samples_per_channel) {
diff --git a/modules/audio_processing/aec_dump/capture_stream_info.h b/modules/audio_processing/aec_dump/capture_stream_info.h index 572990c..bbbbb22 100644 --- a/modules/audio_processing/aec_dump/capture_stream_info.h +++ b/modules/audio_processing/aec_dump/capture_stream_info.h
@@ -11,10 +11,13 @@ #ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_CAPTURE_STREAM_INFO_H_ #define MODULES_AUDIO_PROCESSING_AEC_DUMP_CAPTURE_STREAM_INFO_H_ +#include <cstdint> #include <memory> #include <utility> +#include "api/audio/audio_view.h" #include "modules/audio_processing/include/aec_dump.h" +#include "modules/audio_processing/include/audio_frame_view.h" // Files generated at build-time by the protobuf compiler. #ifdef WEBRTC_ANDROID_PLATFORM_BUILD @@ -33,7 +36,9 @@ ~CaptureStreamInfo() = default; void AddInput(const AudioFrameView<const float>& src); + void AddInputChannel(MonoView<const float> channel); void AddOutput(const AudioFrameView<const float>& src); + void AddOutputChannel(MonoView<const float> channel); void AddInput(const int16_t* const data, int num_channels,
diff --git a/modules/audio_processing/aec_dump/mock_aec_dump.h b/modules/audio_processing/aec_dump/mock_aec_dump.h index b396739..c192d46 100644 --- a/modules/audio_processing/aec_dump/mock_aec_dump.h +++ b/modules/audio_processing/aec_dump/mock_aec_dump.h
@@ -11,9 +11,12 @@ #ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_MOCK_AEC_DUMP_H_ #define MODULES_AUDIO_PROCESSING_AEC_DUMP_MOCK_AEC_DUMP_H_ -#include <memory> +#include <cstdint> +#include "api/audio/audio_processing.h" +#include "api/audio/audio_view.h" #include "modules/audio_processing/include/aec_dump.h" +#include "modules/audio_processing/include/audio_frame_view.h" #include "test/gmock.h" namespace webrtc { @@ -35,10 +38,18 @@ (const AudioFrameView<const float>& src), (override)); MOCK_METHOD(void, + AddCaptureStreamInput, + (MonoView<const float> channel), + (override)); + MOCK_METHOD(void, AddCaptureStreamOutput, (const AudioFrameView<const float>& src), (override)); MOCK_METHOD(void, + AddCaptureStreamOutput, + (MonoView<const float> channel), + (override)); + MOCK_METHOD(void, AddCaptureStreamInput, (const int16_t* const data, int num_channels, @@ -66,6 +77,12 @@ WriteRenderStreamMessage, (const AudioFrameView<const float>& src), (override)); + MOCK_METHOD(void, + WriteRenderStreamMessage, + (const float* const* data, + int num_channels, + int samples_per_channel), + (override)); MOCK_METHOD(void, WriteConfig, (const InternalAPMConfig& config), (override));
diff --git a/modules/audio_processing/agc2/BUILD.gn b/modules/audio_processing/agc2/BUILD.gn index 8660950..e9d1743 100644 --- a/modules/audio_processing/agc2/BUILD.gn +++ b/modules/audio_processing/agc2/BUILD.gn
@@ -196,6 +196,7 @@ ] visibility = [ + "..:audio_processing", "..:gain_controller2", "./*", ]
diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index dfca465..b5b43e2 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc
@@ -11,30 +11,54 @@ #include "modules/audio_processing/audio_processing_impl.h" #include <algorithm> +#include <array> +#include <atomic> #include <cstdint> +#include <cstdio> #include <cstring> #include <memory> #include <optional> #include <string> -#include <type_traits> #include <utility> +#include <vector> #include "absl/base/nullability.h" #include "absl/strings/string_view.h" #include "api/array_view.h" -#include "api/audio/audio_frame.h" +#include "api/audio/audio_processing.h" +#include "api/audio/audio_processing_statistics.h" +#include "api/audio/audio_view.h" +#include "api/audio/echo_canceller3_config.h" +#include "api/audio/echo_control.h" #include "api/environment/environment.h" #include "api/field_trials_view.h" +#include "api/scoped_refptr.h" #include "api/task_queue/task_queue_base.h" #include "common_audio/audio_converter.h" #include "common_audio/include/audio_util.h" +#include "modules/audio_processing/aec3/echo_canceller3.h" #include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "modules/audio_processing/agc/agc_manager_direct.h" +#include "modules/audio_processing/agc/gain_control.h" +#include "modules/audio_processing/agc2/input_volume_controller.h" +#include "modules/audio_processing/agc2/input_volume_stats_reporter.h" #include "modules/audio_processing/audio_buffer.h" -#include "modules/audio_processing/include/audio_frame_view.h" +#include "modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h" +#include "modules/audio_processing/echo_control_mobile_impl.h" +#include "modules/audio_processing/gain_control_impl.h" +#include "modules/audio_processing/gain_controller2.h" +#include "modules/audio_processing/high_pass_filter.h" +#include "modules/audio_processing/include/aec_dump.h" #include "modules/audio_processing/logging/apm_data_dumper.h" +#include "modules/audio_processing/ns/noise_suppressor.h" +#include "modules/audio_processing/ns/ns_config.h" +#include "modules/audio_processing/render_queue_item_verifier.h" +#include "modules/audio_processing/rms_level.h" #include "rtc_base/checks.h" #include "rtc_base/denormal_disabler.h" #include "rtc_base/logging.h" +#include "rtc_base/swap_queue.h" +#include "rtc_base/synchronization/mutex.h" #include "rtc_base/time_utils.h" #include "rtc_base/trace_event.h" #include "system_wrappers/include/metrics.h" @@ -1587,8 +1611,7 @@ formats_.api_format.reverse_input_stream().num_frames(); const size_t num_channels = formats_.api_format.reverse_input_stream().num_channels(); - aec_dump_->WriteRenderStreamMessage( - AudioFrameView<const float>(src, num_channels, channel_size)); + aec_dump_->WriteRenderStreamMessage(src, num_channels, channel_size); } render_.render_audio->CopyFrom(src, formats_.api_format.reverse_input_stream()); @@ -2201,8 +2224,10 @@ const size_t channel_size = formats_.api_format.input_stream().num_frames(); const size_t num_channels = formats_.api_format.input_stream().num_channels(); - aec_dump_->AddCaptureStreamInput( - AudioFrameView<const float>(src, num_channels, channel_size)); + for (size_t ch = 0u; ch < num_channels; ++ch) { + aec_dump_->AddCaptureStreamInput( + MonoView<const float>(src[ch], channel_size)); + } RecordAudioProcessingState(); } @@ -2224,8 +2249,11 @@ const size_t channel_size = formats_.api_format.output_stream().num_frames(); const size_t num_channels = formats_.api_format.output_stream().num_channels(); - aec_dump_->AddCaptureStreamOutput(AudioFrameView<const float>( - processed_capture_stream, num_channels, channel_size)); + for (size_t ch = 0u; ch < num_channels; ++ch) { + aec_dump_->AddCaptureStreamOutput( + MonoView<const float>(processed_capture_stream[ch], channel_size)); + } + aec_dump_->WriteCaptureStreamMessage(); }
diff --git a/modules/audio_processing/include/aec_dump.h b/modules/audio_processing/include/aec_dump.h index 532fa21..4e273d4 100644 --- a/modules/audio_processing/include/aec_dump.h +++ b/modules/audio_processing/include/aec_dump.h
@@ -18,6 +18,7 @@ #include "absl/base/attributes.h" #include "api/audio/audio_processing.h" +#include "api/audio/audio_view.h" #include "modules/audio_processing/include/audio_frame_view.h" namespace webrtc { @@ -87,8 +88,10 @@ // by a WriteCaptureStreamMessage call. virtual void AddCaptureStreamInput( const AudioFrameView<const float>& src) = 0; + virtual void AddCaptureStreamInput(MonoView<const float> channel) = 0; virtual void AddCaptureStreamOutput( const AudioFrameView<const float>& src) = 0; + virtual void AddCaptureStreamOutput(MonoView<const float> channel) = 0; virtual void AddCaptureStreamInput(const int16_t* const data, int num_channels, int samples_per_channel) = 0; @@ -104,6 +107,9 @@ int samples_per_channel) = 0; virtual void WriteRenderStreamMessage( const AudioFrameView<const float>& src) = 0; + virtual void WriteRenderStreamMessage(const float* const* data, + int num_channels, + int samples_per_channel) = 0; virtual void WriteRuntimeSetting( const AudioProcessing::RuntimeSetting& runtime_setting) = 0;