niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 1 | /* |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 11 | #ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |
| 12 | #define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 13 | |
Alejandro Luebs | cb3f9bd | 2015-10-30 01:21:34 | [diff] [blame] | 14 | // MSVC++ requires this to be set before any other includes to get M_PI. |
Patrik Höglund | 3ff90f1 | 2017-12-12 13:41:53 | [diff] [blame] | 15 | #ifndef _USE_MATH_DEFINES |
Alejandro Luebs | cb3f9bd | 2015-10-30 01:21:34 | [diff] [blame] | 16 | #define _USE_MATH_DEFINES |
Patrik Höglund | 3ff90f1 | 2017-12-12 13:41:53 | [diff] [blame] | 17 | #endif |
Alejandro Luebs | cb3f9bd | 2015-10-30 01:21:34 | [diff] [blame] | 18 | |
| 19 | #include <math.h> |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 | [diff] [blame] | 20 | #include <stddef.h> // size_t |
Yves Gerey | 665174f | 2018-06-19 13:03:05 | [diff] [blame] | 21 | #include <stdio.h> // FILE |
peah | 8cee56f | 2017-08-25 05:36:53 | [diff] [blame] | 22 | #include <string.h> |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 23 | |
aluebs@webrtc.org | fb7a039 | 2015-01-05 21:58:58 | [diff] [blame] | 24 | #include <vector> |
ajm@google.com | 22e6515 | 2011-07-18 18:03:01 | [diff] [blame] | 25 | |
Danil Chapovalov | 1ecf29c | 2024-01-09 10:52:10 | [diff] [blame] | 26 | #include "absl/base/nullability.h" |
Ali Tofigh | 1fa87c4 | 2022-07-25 20:07:08 | [diff] [blame] | 27 | #include "absl/strings/string_view.h" |
Danil Chapovalov | db9f7ab | 2018-06-19 08:50:11 | [diff] [blame] | 28 | #include "absl/types/optional.h" |
Sam Zackrisson | ab866a2 | 2020-05-07 11:07:49 | [diff] [blame] | 29 | #include "api/array_view.h" |
Gustaf Ullberg | bffa300 | 2018-02-14 14:12:00 | [diff] [blame] | 30 | #include "api/audio/echo_canceller3_config.h" |
Gustaf Ullberg | fd4ce50 | 2018-02-15 09:09:09 | [diff] [blame] | 31 | #include "api/audio/echo_control.h" |
Harald Alvestrand | 78f905e | 2023-11-02 14:09:26 | [diff] [blame] | 32 | #include "api/ref_count.h" |
Mirko Bonadei | d970807 | 2019-01-25 19:26:48 | [diff] [blame] | 33 | #include "api/scoped_refptr.h" |
Danil Chapovalov | 1ecf29c | 2024-01-09 10:52:10 | [diff] [blame] | 34 | #include "api/task_queue/task_queue_base.h" |
Ivo Creusen | 56d46090 | 2017-11-24 16:29:59 | [diff] [blame] | 35 | #include "modules/audio_processing/include/audio_processing_statistics.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 36 | #include "rtc_base/arraysize.h" |
Per Åhgren | 09e9a83 | 2020-05-11 09:03:47 | [diff] [blame] | 37 | #include "rtc_base/system/file_wrapper.h" |
Mirko Bonadei | 3d25530 | 2018-10-11 08:50:45 | [diff] [blame] | 38 | #include "rtc_base/system/rtc_export.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 39 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 40 | namespace webrtc { |
| 41 | |
aleloi | 868f32f | 2017-05-23 14:20:05 | [diff] [blame] | 42 | class AecDump; |
Sam Zackrisson | 0beac58 | 2017-09-25 10:04:02 | [diff] [blame] | 43 | class AudioBuffer; |
Michael Graczyk | dfa3605 | 2015-03-25 23:37:27 | [diff] [blame] | 44 | |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 45 | class StreamConfig; |
| 46 | class ProcessingConfig; |
| 47 | |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 48 | class EchoDetector; |
Valeriia Nemychnikova | f06eb57 | 2018-08-29 08:37:09 | [diff] [blame] | 49 | class CustomAudioAnalyzer; |
Alex Loiko | 5825aa6 | 2017-12-18 15:02:40 | [diff] [blame] | 50 | class CustomProcessing; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 51 | |
| 52 | // The Audio Processing Module (APM) provides a collection of voice processing |
| 53 | // components designed for real-time communications software. |
| 54 | // |
| 55 | // APM operates on two audio streams on a frame-by-frame basis. Frames of the |
| 56 | // primary stream, on which all processing is applied, are passed to |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 57 | // `ProcessStream()`. Frames of the reverse direction stream are passed to |
| 58 | // `ProcessReverseStream()`. On the client-side, this will typically be the |
aluebs | b031955 | 2016-03-18 03:39:53 | [diff] [blame] | 59 | // near-end (capture) and far-end (render) streams, respectively. APM should be |
| 60 | // placed in the signal chain as close to the audio hardware abstraction layer |
| 61 | // (HAL) as possible. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 62 | // |
| 63 | // On the server-side, the reverse stream will normally not be used, with |
| 64 | // processing occurring on each incoming stream. |
| 65 | // |
| 66 | // Component interfaces follow a similar pattern and are accessed through |
| 67 | // corresponding getters in APM. All components are disabled at create-time, |
| 68 | // with default settings that are recommended for most situations. New settings |
| 69 | // can be applied without enabling a component. Enabling a component triggers |
| 70 | // memory allocation and initialization to allow it to start processing the |
| 71 | // streams. |
| 72 | // |
| 73 | // Thread safety is provided with the following assumptions to reduce locking |
| 74 | // overhead: |
| 75 | // 1. The stream getters and setters are called from the same thread as |
| 76 | // ProcessStream(). More precisely, stream functions are never called |
| 77 | // concurrently with ProcessStream(). |
| 78 | // 2. Parameter getters are never called concurrently with the corresponding |
| 79 | // setter. |
| 80 | // |
Sam Zackrisson | 3bd444f | 2022-08-03 12:37:00 | [diff] [blame] | 81 | // APM accepts only linear PCM audio data in chunks of ~10 ms (see |
Sam Zackrisson | 5dd5482 | 2022-11-17 10:26:58 | [diff] [blame] | 82 | // AudioProcessing::GetFrameSize() for details) and sample rates ranging from |
| 83 | // 8000 Hz to 384000 Hz. The int16 interfaces use interleaved data, while the |
| 84 | // float interfaces use deinterleaved data. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 85 | // |
| 86 | // Usage example, omitting error checking: |
Sam Zackrisson | 5dd5482 | 2022-11-17 10:26:58 | [diff] [blame] | 87 | // rtc::scoped_refptr<AudioProcessing> apm = AudioProcessingBuilder().Create(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 88 | // |
peah | 88ac853 | 2016-09-12 23:47:25 | [diff] [blame] | 89 | // AudioProcessing::Config config; |
Sam Zackrisson | cdf0e6d | 2018-09-17 09:05:17 | [diff] [blame] | 90 | // config.echo_canceller.enabled = true; |
| 91 | // config.echo_canceller.mobile_mode = false; |
Sam Zackrisson | 41478c7 | 2019-10-15 08:10:26 | [diff] [blame] | 92 | // |
| 93 | // config.gain_controller1.enabled = true; |
| 94 | // config.gain_controller1.mode = |
| 95 | // AudioProcessing::Config::GainController1::kAdaptiveAnalog; |
| 96 | // config.gain_controller1.analog_level_minimum = 0; |
| 97 | // config.gain_controller1.analog_level_maximum = 255; |
| 98 | // |
Sam Zackrisson | ab1aee0 | 2018-03-05 14:59:06 | [diff] [blame] | 99 | // config.gain_controller2.enabled = true; |
Sam Zackrisson | 41478c7 | 2019-10-15 08:10:26 | [diff] [blame] | 100 | // |
| 101 | // config.high_pass_filter.enabled = true; |
| 102 | // |
peah | 88ac853 | 2016-09-12 23:47:25 | [diff] [blame] | 103 | // apm->ApplyConfig(config) |
| 104 | // |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 105 | // // Start a voice call... |
| 106 | // |
| 107 | // // ... Render frame arrives bound for the audio HAL ... |
aluebs | b031955 | 2016-03-18 03:39:53 | [diff] [blame] | 108 | // apm->ProcessReverseStream(render_frame); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 109 | // |
| 110 | // // ... Capture frame arrives from the audio HAL ... |
| 111 | // // Call required set_stream_ functions. |
| 112 | // apm->set_stream_delay_ms(delay_ms); |
Sam Zackrisson | 41478c7 | 2019-10-15 08:10:26 | [diff] [blame] | 113 | // apm->set_stream_analog_level(analog_level); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 114 | // |
| 115 | // apm->ProcessStream(capture_frame); |
| 116 | // |
| 117 | // // Call required stream_ functions. |
Sam Zackrisson | 41478c7 | 2019-10-15 08:10:26 | [diff] [blame] | 118 | // analog_level = apm->recommended_stream_analog_level(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 119 | // has_voice = apm->stream_has_voice(); |
| 120 | // |
Hua, Chunbo | e61a40e | 2021-01-08 08:34:49 | [diff] [blame] | 121 | // // Repeat render and capture processing for the duration of the call... |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 122 | // // Start a new call... |
| 123 | // apm->Initialize(); |
| 124 | // |
| 125 | // // Close the application... |
Sam Zackrisson | 5dd5482 | 2022-11-17 10:26:58 | [diff] [blame] | 126 | // apm.reset(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 127 | // |
Harald Alvestrand | 78f905e | 2023-11-02 14:09:26 | [diff] [blame] | 128 | class RTC_EXPORT AudioProcessing : public RefCountInterface { |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 129 | public: |
peah | 88ac853 | 2016-09-12 23:47:25 | [diff] [blame] | 130 | // The struct below constitutes the new parameter scheme for the audio |
| 131 | // processing. It is being introduced gradually and until it is fully |
| 132 | // introduced, it is prone to change. |
| 133 | // TODO(peah): Remove this comment once the new config scheme is fully rolled |
| 134 | // out. |
| 135 | // |
| 136 | // The parameters and behavior of the audio processing module are controlled |
| 137 | // by changing the default values in the AudioProcessing::Config struct. |
| 138 | // The config is applied by passing the struct to the ApplyConfig method. |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 139 | // |
| 140 | // This config is intended to be used during setup, and to enable/disable |
| 141 | // top-level processing effects. Use during processing may cause undesired |
| 142 | // submodule resets, affecting the audio quality. Use the RuntimeSetting |
| 143 | // construct for runtime configuration. |
Mirko Bonadei | d4002a7 | 2019-11-12 19:11:48 | [diff] [blame] | 144 | struct RTC_EXPORT Config { |
Per Åhgren | fcbe407 | 2019-09-14 22:27:58 | [diff] [blame] | 145 | // Sets the properties of the audio processing pipeline. |
Mirko Bonadei | d4002a7 | 2019-11-12 19:11:48 | [diff] [blame] | 146 | struct RTC_EXPORT Pipeline { |
Alessio Bazzica | 504bd59 | 2022-12-01 12:26:26 | [diff] [blame] | 147 | // Ways to downmix a multi-channel track to mono. |
| 148 | enum class DownmixMethod { |
| 149 | kAverageChannels, // Average across channels. |
| 150 | kUseFirstChannel // Use the first channel. |
| 151 | }; |
| 152 | |
Per Åhgren | fcbe407 | 2019-09-14 22:27:58 | [diff] [blame] | 153 | // Maximum allowed processing rate used internally. May only be set to |
Per Åhgren | 68c225d | 2021-01-21 22:03:32 | [diff] [blame] | 154 | // 32000 or 48000 and any differing values will be treated as 48000. |
| 155 | int maximum_internal_processing_rate = 48000; |
Per Åhgren | e14cb99 | 2019-11-27 08:34:22 | [diff] [blame] | 156 | // Allow multi-channel processing of render audio. |
| 157 | bool multi_channel_render = false; |
| 158 | // Allow multi-channel processing of capture audio when AEC3 is active |
| 159 | // or a custom AEC is injected.. |
| 160 | bool multi_channel_capture = false; |
Alessio Bazzica | 504bd59 | 2022-12-01 12:26:26 | [diff] [blame] | 161 | // Indicates how to downmix multi-channel capture audio to mono (when |
| 162 | // needed). |
| 163 | DownmixMethod capture_downmix_method = DownmixMethod::kAverageChannels; |
Per Åhgren | fcbe407 | 2019-09-14 22:27:58 | [diff] [blame] | 164 | } pipeline; |
| 165 | |
Sam Zackrisson | 2351313 | 2019-01-11 14:10:32 | [diff] [blame] | 166 | // Enabled the pre-amplifier. It amplifies the capture signal |
| 167 | // before any other processing is done. |
Per Åhgren | db5d728 | 2021-03-15 16:31:04 | [diff] [blame] | 168 | // TODO(webrtc:5298): Deprecate and use the pre-gain functionality in |
| 169 | // capture_level_adjustment instead. |
Sam Zackrisson | 2351313 | 2019-01-11 14:10:32 | [diff] [blame] | 170 | struct PreAmplifier { |
| 171 | bool enabled = false; |
Alessio Bazzica | 841d74e | 2021-03-31 13:04:03 | [diff] [blame] | 172 | float fixed_gain_factor = 1.0f; |
Sam Zackrisson | 2351313 | 2019-01-11 14:10:32 | [diff] [blame] | 173 | } pre_amplifier; |
| 174 | |
Per Åhgren | db5d728 | 2021-03-15 16:31:04 | [diff] [blame] | 175 | // Functionality for general level adjustment in the capture pipeline. This |
| 176 | // should not be used together with the legacy PreAmplifier functionality. |
| 177 | struct CaptureLevelAdjustment { |
| 178 | bool operator==(const CaptureLevelAdjustment& rhs) const; |
| 179 | bool operator!=(const CaptureLevelAdjustment& rhs) const { |
| 180 | return !(*this == rhs); |
| 181 | } |
| 182 | bool enabled = false; |
| 183 | // The `pre_gain_factor` scales the signal before any processing is done. |
Alessio Bazzica | 841d74e | 2021-03-31 13:04:03 | [diff] [blame] | 184 | float pre_gain_factor = 1.0f; |
Per Åhgren | db5d728 | 2021-03-15 16:31:04 | [diff] [blame] | 185 | // The `post_gain_factor` scales the signal after all processing is done. |
Alessio Bazzica | 841d74e | 2021-03-31 13:04:03 | [diff] [blame] | 186 | float post_gain_factor = 1.0f; |
Per Åhgren | db5d728 | 2021-03-15 16:31:04 | [diff] [blame] | 187 | struct AnalogMicGainEmulation { |
| 188 | bool operator==(const AnalogMicGainEmulation& rhs) const; |
| 189 | bool operator!=(const AnalogMicGainEmulation& rhs) const { |
| 190 | return !(*this == rhs); |
| 191 | } |
| 192 | bool enabled = false; |
| 193 | // Initial analog gain level to use for the emulated analog gain. Must |
| 194 | // be in the range [0...255]. |
| 195 | int initial_level = 255; |
| 196 | } analog_mic_gain_emulation; |
| 197 | } capture_level_adjustment; |
| 198 | |
Sam Zackrisson | 2351313 | 2019-01-11 14:10:32 | [diff] [blame] | 199 | struct HighPassFilter { |
| 200 | bool enabled = false; |
Per Åhgren | c042425 | 2019-12-10 12:04:15 | [diff] [blame] | 201 | bool apply_in_full_band = true; |
Sam Zackrisson | 2351313 | 2019-01-11 14:10:32 | [diff] [blame] | 202 | } high_pass_filter; |
| 203 | |
Sam Zackrisson | 8b5d2cc | 2018-07-27 11:27:23 | [diff] [blame] | 204 | struct EchoCanceller { |
| 205 | bool enabled = false; |
| 206 | bool mobile_mode = false; |
Per Åhgren | c20a19c | 2019-11-13 10:12:29 | [diff] [blame] | 207 | bool export_linear_aec_output = false; |
Per Åhgren | b810646 | 2019-12-04 07:34:12 | [diff] [blame] | 208 | // Enforce the highpass filter to be on (has no effect for the mobile |
| 209 | // mode). |
Per Åhgren | bcce453 | 2019-12-03 12:52:40 | [diff] [blame] | 210 | bool enforce_high_pass_filtering = true; |
Sam Zackrisson | 8b5d2cc | 2018-07-27 11:27:23 | [diff] [blame] | 211 | } echo_canceller; |
| 212 | |
Sam Zackrisson | 2351313 | 2019-01-11 14:10:32 | [diff] [blame] | 213 | // Enables background noise suppression. |
| 214 | struct NoiseSuppression { |
peah | 8271d04 | 2016-11-22 15:24:52 | [diff] [blame] | 215 | bool enabled = false; |
Sam Zackrisson | 2351313 | 2019-01-11 14:10:32 | [diff] [blame] | 216 | enum Level { kLow, kModerate, kHigh, kVeryHigh }; |
| 217 | Level level = kModerate; |
Per Åhgren | 2e8e1c6 | 2019-12-19 23:42:22 | [diff] [blame] | 218 | bool analyze_linear_aec_output_when_available = false; |
Sam Zackrisson | 2351313 | 2019-01-11 14:10:32 | [diff] [blame] | 219 | } noise_suppression; |
peah | e0eae3c | 2016-12-14 09:16:23 | [diff] [blame] | 220 | |
Per Åhgren | c073471 | 2020-01-02 14:15:36 | [diff] [blame] | 221 | // Enables transient suppression. |
| 222 | struct TransientSuppression { |
| 223 | bool enabled = false; |
| 224 | } transient_suppression; |
| 225 | |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 226 | // Enables automatic gain control (AGC) functionality. |
| 227 | // The automatic gain control (AGC) component brings the signal to an |
| 228 | // appropriate range. This is done by applying a digital gain directly and, |
| 229 | // in the analog mode, prescribing an analog gain to be applied at the audio |
| 230 | // HAL. |
| 231 | // Recommended to be enabled on the client-side. |
Alessio Bazzica | dfc11d5 | 2021-05-07 09:58:11 | [diff] [blame] | 232 | struct RTC_EXPORT GainController1 { |
Alessio Bazzica | 3438a93 | 2020-10-14 10:47:50 | [diff] [blame] | 233 | bool operator==(const GainController1& rhs) const; |
| 234 | bool operator!=(const GainController1& rhs) const { |
| 235 | return !(*this == rhs); |
| 236 | } |
| 237 | |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 238 | bool enabled = false; |
| 239 | enum Mode { |
| 240 | // Adaptive mode intended for use if an analog volume control is |
| 241 | // available on the capture device. It will require the user to provide |
| 242 | // coupling between the OS mixer controls and AGC through the |
| 243 | // stream_analog_level() functions. |
| 244 | // It consists of an analog gain prescription for the audio device and a |
| 245 | // digital compression stage. |
| 246 | kAdaptiveAnalog, |
| 247 | // Adaptive mode intended for situations in which an analog volume |
| 248 | // control is unavailable. It operates in a similar fashion to the |
| 249 | // adaptive analog mode, but with scaling instead applied in the digital |
| 250 | // domain. As with the analog mode, it additionally uses a digital |
| 251 | // compression stage. |
| 252 | kAdaptiveDigital, |
| 253 | // Fixed mode which enables only the digital compression stage also used |
| 254 | // by the two adaptive modes. |
| 255 | // It is distinguished from the adaptive modes by considering only a |
| 256 | // short time-window of the input signal. It applies a fixed gain |
| 257 | // through most of the input level range, and compresses (gradually |
| 258 | // reduces gain with increasing level) the input signal at higher |
| 259 | // levels. This mode is preferred on embedded devices where the capture |
| 260 | // signal level is predictable, so that a known gain can be applied. |
| 261 | kFixedDigital |
| 262 | }; |
| 263 | Mode mode = kAdaptiveAnalog; |
| 264 | // Sets the target peak level (or envelope) of the AGC in dBFs (decibels |
| 265 | // from digital full-scale). The convention is to use positive values. For |
| 266 | // instance, passing in a value of 3 corresponds to -3 dBFs, or a target |
| 267 | // level 3 dB below full-scale. Limited to [0, 31]. |
| 268 | int target_level_dbfs = 3; |
| 269 | // Sets the maximum gain the digital compression stage may apply, in dB. A |
| 270 | // higher number corresponds to greater compression, while a value of 0 |
| 271 | // will leave the signal uncompressed. Limited to [0, 90]. |
| 272 | // For updates after APM setup, use a RuntimeSetting instead. |
| 273 | int compression_gain_db = 9; |
| 274 | // When enabled, the compression stage will hard limit the signal to the |
| 275 | // target level. Otherwise, the signal will be compressed but not limited |
| 276 | // above the target level. |
| 277 | bool enable_limiter = true; |
Per Åhgren | 0695df1 | 2020-01-13 13:43:13 | [diff] [blame] | 278 | |
| 279 | // Enables the analog gain controller functionality. |
| 280 | struct AnalogGainController { |
| 281 | bool enabled = true; |
Alessio Bazzica | 7afd698 | 2022-10-13 15:15:36 | [diff] [blame] | 282 | // TODO(bugs.webrtc.org/7494): Deprecated. Stop using and remove. |
| 283 | int startup_min_volume = 0; |
Per Åhgren | 0695df1 | 2020-01-13 13:43:13 | [diff] [blame] | 284 | // Lowest analog microphone level that will be applied in response to |
| 285 | // clipping. |
Alessio Bazzica | 488f669 | 2022-10-13 11:06:05 | [diff] [blame] | 286 | int clipped_level_min = 70; |
Alessio Bazzica | 866caeb | 2022-07-19 10:18:38 | [diff] [blame] | 287 | // If true, an adaptive digital gain is applied. |
Per Åhgren | 0695df1 | 2020-01-13 13:43:13 | [diff] [blame] | 288 | bool enable_digital_adaptive = true; |
Hanna Silen | b8dc7fa | 2021-05-20 15:37:56 | [diff] [blame] | 289 | // Amount the microphone level is lowered with every clipping event. |
| 290 | // Limited to (0, 255]. |
| 291 | int clipped_level_step = 15; |
| 292 | // Proportion of clipped samples required to declare a clipping event. |
| 293 | // Limited to (0.f, 1.f). |
| 294 | float clipped_ratio_threshold = 0.1f; |
| 295 | // Time in frames to wait after a clipping event before checking again. |
| 296 | // Limited to values higher than 0. |
| 297 | int clipped_wait_frames = 300; |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 298 | |
| 299 | // Enables clipping prediction functionality. |
| 300 | struct ClippingPredictor { |
| 301 | bool enabled = false; |
| 302 | enum Mode { |
Alessio Bazzica | b237a87 | 2021-06-11 10:37:54 | [diff] [blame] | 303 | // Clipping event prediction mode with fixed step estimation. |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 304 | kClippingEventPrediction, |
Alessio Bazzica | b237a87 | 2021-06-11 10:37:54 | [diff] [blame] | 305 | // Clipped peak estimation mode with adaptive step estimation. |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 306 | kAdaptiveStepClippingPeakPrediction, |
Alessio Bazzica | b237a87 | 2021-06-11 10:37:54 | [diff] [blame] | 307 | // Clipped peak estimation mode with fixed step estimation. |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 308 | kFixedStepClippingPeakPrediction, |
| 309 | }; |
| 310 | Mode mode = kClippingEventPrediction; |
Alessio Bazzica | b237a87 | 2021-06-11 10:37:54 | [diff] [blame] | 311 | // Number of frames in the sliding analysis window. |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 312 | int window_length = 5; |
Alessio Bazzica | b237a87 | 2021-06-11 10:37:54 | [diff] [blame] | 313 | // Number of frames in the sliding reference window. |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 314 | int reference_window_length = 5; |
Alessio Bazzica | b237a87 | 2021-06-11 10:37:54 | [diff] [blame] | 315 | // Reference window delay (unit: number of frames). |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 316 | int reference_window_delay = 5; |
Alessio Bazzica | b237a87 | 2021-06-11 10:37:54 | [diff] [blame] | 317 | // Clipping prediction threshold (dBFS). |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 318 | float clipping_threshold = -1.0f; |
| 319 | // Crest factor drop threshold (dB). |
| 320 | float crest_factor_margin = 3.0f; |
Alessio Bazzica | 42dacda | 2021-06-17 15:18:46 | [diff] [blame] | 321 | // If true, the recommended clipped level step is used to modify the |
| 322 | // analog gain. Otherwise, the predictor runs without affecting the |
| 323 | // analog gain. |
| 324 | bool use_predicted_step = true; |
Hanna Silen | a43953a | 2021-06-02 15:13:24 | [diff] [blame] | 325 | } clipping_predictor; |
Per Åhgren | 0695df1 | 2020-01-13 13:43:13 | [diff] [blame] | 326 | } analog_gain_controller; |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 327 | } gain_controller1; |
| 328 | |
Alessio Bazzica | 4366c54 | 2022-12-05 15:31:16 | [diff] [blame] | 329 | // Parameters for AGC2, an Automatic Gain Control (AGC) sub-module which |
| 330 | // replaces the AGC sub-module parametrized by `gain_controller1`. |
| 331 | // AGC2 brings the captured audio signal to the desired level by combining |
| 332 | // three different controllers (namely, input volume controller, adapative |
| 333 | // digital controller and fixed digital controller) and a limiter. |
| 334 | // TODO(bugs.webrtc.org:7494): Name `GainController` when AGC1 removed. |
Alessio Bazzica | dfc11d5 | 2021-05-07 09:58:11 | [diff] [blame] | 335 | struct RTC_EXPORT GainController2 { |
Alessio Bazzica | 3438a93 | 2020-10-14 10:47:50 | [diff] [blame] | 336 | bool operator==(const GainController2& rhs) const; |
| 337 | bool operator!=(const GainController2& rhs) const { |
| 338 | return !(*this == rhs); |
| 339 | } |
| 340 | |
Alessio Bazzica | 4366c54 | 2022-12-05 15:31:16 | [diff] [blame] | 341 | // AGC2 must be created if and only if `enabled` is true. |
alessiob | 3ec96df | 2017-05-22 13:57:06 | [diff] [blame] | 342 | bool enabled = false; |
Alessio Bazzica | 4366c54 | 2022-12-05 15:31:16 | [diff] [blame] | 343 | |
| 344 | // Parameters for the input volume controller, which adjusts the input |
| 345 | // volume applied when the audio is captured (e.g., microphone volume on |
| 346 | // a soundcard, input volume on HAL). |
| 347 | struct InputVolumeController { |
| 348 | bool operator==(const InputVolumeController& rhs) const; |
| 349 | bool operator!=(const InputVolumeController& rhs) const { |
| 350 | return !(*this == rhs); |
| 351 | } |
| 352 | bool enabled = false; |
| 353 | } input_volume_controller; |
| 354 | |
| 355 | // Parameters for the adaptive digital controller, which adjusts and |
| 356 | // applies a digital gain after echo cancellation and after noise |
| 357 | // suppression. |
Alessio Bazzica | dfc11d5 | 2021-05-07 09:58:11 | [diff] [blame] | 358 | struct RTC_EXPORT AdaptiveDigital { |
Alessio Bazzica | a2efd15 | 2021-04-29 14:17:49 | [diff] [blame] | 359 | bool operator==(const AdaptiveDigital& rhs) const; |
| 360 | bool operator!=(const AdaptiveDigital& rhs) const { |
| 361 | return !(*this == rhs); |
| 362 | } |
Alessio Bazzica | 8da7b35 | 2018-11-21 09:50:58 | [diff] [blame] | 363 | bool enabled = false; |
Alessio Bazzica | a850e6c | 2021-10-04 11:35:55 | [diff] [blame] | 364 | float headroom_db = 6.0f; |
Alessio Bazzica | a850e6c | 2021-10-04 11:35:55 | [diff] [blame] | 365 | float max_gain_db = 30.0f; |
| 366 | float initial_gain_db = 8.0f; |
Alessio Bazzica | 841d74e | 2021-03-31 13:04:03 | [diff] [blame] | 367 | float max_gain_change_db_per_second = 3.0f; |
Alessio Bazzica | 980c460 | 2021-04-14 17:09:17 | [diff] [blame] | 368 | float max_output_noise_level_dbfs = -50.0f; |
Alessio Bazzica | 1e2542f | 2018-11-13 13:44:15 | [diff] [blame] | 369 | } adaptive_digital; |
Hanna Silen | 9f06ef1 | 2022-11-01 16:17:54 | [diff] [blame] | 370 | |
Alessio Bazzica | 4366c54 | 2022-12-05 15:31:16 | [diff] [blame] | 371 | // Parameters for the fixed digital controller, which applies a fixed |
| 372 | // digital gain after the adaptive digital controller and before the |
| 373 | // limiter. |
| 374 | struct FixedDigital { |
| 375 | // By setting `gain_db` to a value greater than zero, the limiter can be |
| 376 | // turned into a compressor that first applies a fixed gain. |
| 377 | float gain_db = 0.0f; |
| 378 | } fixed_digital; |
alessiob | 3ec96df | 2017-05-22 13:57:06 | [diff] [blame] | 379 | } gain_controller2; |
peah | 8cee56f | 2017-08-25 05:36:53 | [diff] [blame] | 380 | |
Artem Titov | 59bbd65 | 2019-08-02 09:31:37 | [diff] [blame] | 381 | std::string ToString() const; |
peah | 88ac853 | 2016-09-12 23:47:25 | [diff] [blame] | 382 | }; |
| 383 | |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 384 | // Specifies the properties of a setting to be passed to AudioProcessing at |
| 385 | // runtime. |
| 386 | class RuntimeSetting { |
| 387 | public: |
Alex Loiko | 73ec019 | 2018-05-15 08:52:28 | [diff] [blame] | 388 | enum class Type { |
| 389 | kNotSpecified, |
| 390 | kCapturePreGain, |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 391 | kCaptureCompressionGain, |
Per Åhgren | 6ee75fd | 2019-04-26 09:33:37 | [diff] [blame] | 392 | kCaptureFixedPostGain, |
Fredrik Hernqvist | ca36285 | 2019-05-10 13:50:02 | [diff] [blame] | 393 | kPlayoutVolumeChange, |
Alessio Bazzica | 7c19a70 | 2019-11-07 12:22:00 | [diff] [blame] | 394 | kCustomRenderProcessingRuntimeSetting, |
Per Åhgren | 552d3e3 | 2020-08-12 06:46:47 | [diff] [blame] | 395 | kPlayoutAudioDeviceChange, |
Per Åhgren | db5d728 | 2021-03-15 16:31:04 | [diff] [blame] | 396 | kCapturePostGain, |
Per Åhgren | 552d3e3 | 2020-08-12 06:46:47 | [diff] [blame] | 397 | kCaptureOutputUsed |
Alessio Bazzica | 7c19a70 | 2019-11-07 12:22:00 | [diff] [blame] | 398 | }; |
| 399 | |
| 400 | // Play-out audio device properties. |
| 401 | struct PlayoutAudioDeviceInfo { |
| 402 | int id; // Identifies the audio device. |
| 403 | int max_volume; // Maximum play-out volume. |
Alex Loiko | 73ec019 | 2018-05-15 08:52:28 | [diff] [blame] | 404 | }; |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 405 | |
Alessio Bazzica | 841d74e | 2021-03-31 13:04:03 | [diff] [blame] | 406 | RuntimeSetting() : type_(Type::kNotSpecified), value_(0.0f) {} |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 407 | ~RuntimeSetting() = default; |
| 408 | |
| 409 | static RuntimeSetting CreateCapturePreGain(float gain) { |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 410 | return {Type::kCapturePreGain, gain}; |
| 411 | } |
| 412 | |
Per Åhgren | db5d728 | 2021-03-15 16:31:04 | [diff] [blame] | 413 | static RuntimeSetting CreateCapturePostGain(float gain) { |
| 414 | return {Type::kCapturePostGain, gain}; |
| 415 | } |
| 416 | |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 417 | // Corresponds to Config::GainController1::compression_gain_db, but for |
| 418 | // runtime configuration. |
| 419 | static RuntimeSetting CreateCompressionGainDb(int gain_db) { |
| 420 | RTC_DCHECK_GE(gain_db, 0); |
| 421 | RTC_DCHECK_LE(gain_db, 90); |
| 422 | return {Type::kCaptureCompressionGain, static_cast<float>(gain_db)}; |
| 423 | } |
| 424 | |
Per Åhgren | 6ee75fd | 2019-04-26 09:33:37 | [diff] [blame] | 425 | // Corresponds to Config::GainController2::fixed_digital::gain_db, but for |
| 426 | // runtime configuration. |
| 427 | static RuntimeSetting CreateCaptureFixedPostGain(float gain_db) { |
Alessio Bazzica | 841d74e | 2021-03-31 13:04:03 | [diff] [blame] | 428 | RTC_DCHECK_GE(gain_db, 0.0f); |
| 429 | RTC_DCHECK_LE(gain_db, 90.0f); |
Per Åhgren | 6ee75fd | 2019-04-26 09:33:37 | [diff] [blame] | 430 | return {Type::kCaptureFixedPostGain, gain_db}; |
| 431 | } |
| 432 | |
Alessio Bazzica | 7c19a70 | 2019-11-07 12:22:00 | [diff] [blame] | 433 | // Creates a runtime setting to notify play-out (aka render) audio device |
| 434 | // changes. |
| 435 | static RuntimeSetting CreatePlayoutAudioDeviceChange( |
| 436 | PlayoutAudioDeviceInfo audio_device) { |
| 437 | return {Type::kPlayoutAudioDeviceChange, audio_device}; |
| 438 | } |
| 439 | |
| 440 | // Creates a runtime setting to notify play-out (aka render) volume changes. |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 441 | // `volume` is the unnormalized volume, the maximum of which |
Fredrik Hernqvist | ca36285 | 2019-05-10 13:50:02 | [diff] [blame] | 442 | static RuntimeSetting CreatePlayoutVolumeChange(int volume) { |
| 443 | return {Type::kPlayoutVolumeChange, volume}; |
| 444 | } |
| 445 | |
Alex Loiko | 73ec019 | 2018-05-15 08:52:28 | [diff] [blame] | 446 | static RuntimeSetting CreateCustomRenderSetting(float payload) { |
| 447 | return {Type::kCustomRenderProcessingRuntimeSetting, payload}; |
| 448 | } |
| 449 | |
Per Åhgren | 652ada5 | 2021-03-03 10:52:44 | [diff] [blame] | 450 | static RuntimeSetting CreateCaptureOutputUsedSetting( |
| 451 | bool capture_output_used) { |
| 452 | return {Type::kCaptureOutputUsed, capture_output_used}; |
Per Åhgren | 552d3e3 | 2020-08-12 06:46:47 | [diff] [blame] | 453 | } |
| 454 | |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 455 | Type type() const { return type_; } |
Alessio Bazzica | 7c19a70 | 2019-11-07 12:22:00 | [diff] [blame] | 456 | // Getters do not return a value but instead modify the argument to protect |
| 457 | // from implicit casting. |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 458 | void GetFloat(float* value) const { |
| 459 | RTC_DCHECK(value); |
Fredrik Hernqvist | ca36285 | 2019-05-10 13:50:02 | [diff] [blame] | 460 | *value = value_.float_value; |
| 461 | } |
| 462 | void GetInt(int* value) const { |
| 463 | RTC_DCHECK(value); |
| 464 | *value = value_.int_value; |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 465 | } |
Per Åhgren | 552d3e3 | 2020-08-12 06:46:47 | [diff] [blame] | 466 | void GetBool(bool* value) const { |
| 467 | RTC_DCHECK(value); |
| 468 | *value = value_.bool_value; |
| 469 | } |
Alessio Bazzica | 7c19a70 | 2019-11-07 12:22:00 | [diff] [blame] | 470 | void GetPlayoutAudioDeviceInfo(PlayoutAudioDeviceInfo* value) const { |
| 471 | RTC_DCHECK(value); |
| 472 | *value = value_.playout_audio_device_info; |
| 473 | } |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 474 | |
| 475 | private: |
| 476 | RuntimeSetting(Type id, float value) : type_(id), value_(value) {} |
Fredrik Hernqvist | ca36285 | 2019-05-10 13:50:02 | [diff] [blame] | 477 | RuntimeSetting(Type id, int value) : type_(id), value_(value) {} |
Alessio Bazzica | 7c19a70 | 2019-11-07 12:22:00 | [diff] [blame] | 478 | RuntimeSetting(Type id, PlayoutAudioDeviceInfo value) |
| 479 | : type_(id), value_(value) {} |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 480 | Type type_; |
Fredrik Hernqvist | ca36285 | 2019-05-10 13:50:02 | [diff] [blame] | 481 | union U { |
| 482 | U() {} |
| 483 | U(int value) : int_value(value) {} |
| 484 | U(float value) : float_value(value) {} |
Alessio Bazzica | 7c19a70 | 2019-11-07 12:22:00 | [diff] [blame] | 485 | U(PlayoutAudioDeviceInfo value) : playout_audio_device_info(value) {} |
Fredrik Hernqvist | ca36285 | 2019-05-10 13:50:02 | [diff] [blame] | 486 | float float_value; |
| 487 | int int_value; |
Per Åhgren | 552d3e3 | 2020-08-12 06:46:47 | [diff] [blame] | 488 | bool bool_value; |
Alessio Bazzica | 7c19a70 | 2019-11-07 12:22:00 | [diff] [blame] | 489 | PlayoutAudioDeviceInfo playout_audio_device_info; |
Fredrik Hernqvist | ca36285 | 2019-05-10 13:50:02 | [diff] [blame] | 490 | } value_; |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 491 | }; |
| 492 | |
peah | a9cc40b | 2017-06-29 15:32:09 | [diff] [blame] | 493 | ~AudioProcessing() override {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 494 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 495 | // Initializes internal states, while retaining all user settings. This |
| 496 | // should be called before beginning to process a new audio stream. However, |
| 497 | // it is not necessary to call before processing the first stream after |
andrew@webrtc.org | ddbb8a2 | 2014-04-22 21:00:04 | [diff] [blame] | 498 | // creation. |
| 499 | // |
| 500 | // It is also not necessary to call if the audio parameters (sample |
andrew@webrtc.org | 60730cf | 2014-01-07 17:45:09 | [diff] [blame] | 501 | // rate and number of channels) have changed. Passing updated parameters |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 502 | // directly to `ProcessStream()` and `ProcessReverseStream()` is permissible. |
andrew@webrtc.org | ddbb8a2 | 2014-04-22 21:00:04 | [diff] [blame] | 503 | // If the parameters are known at init-time though, they may be provided. |
Per Åhgren | 0ade983 | 2020-09-01 21:57:20 | [diff] [blame] | 504 | // TODO(webrtc:5298): Change to return void. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 505 | virtual int Initialize() = 0; |
andrew@webrtc.org | ddbb8a2 | 2014-04-22 21:00:04 | [diff] [blame] | 506 | |
| 507 | // The int16 interfaces require: |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 508 | // - only `NativeRate`s be used |
andrew@webrtc.org | ddbb8a2 | 2014-04-22 21:00:04 | [diff] [blame] | 509 | // - that the input, output and reverse rates must match |
Artem Titov | cfea218 | 2021-08-09 23:22:31 | [diff] [blame] | 510 | // - that `processing_config.output_stream()` matches |
| 511 | // `processing_config.input_stream()`. |
andrew@webrtc.org | ddbb8a2 | 2014-04-22 21:00:04 | [diff] [blame] | 512 | // |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 513 | // The float interfaces accept arbitrary rates and support differing input and |
| 514 | // output layouts, but the output must have either one channel or the same |
| 515 | // number of channels as the input. |
| 516 | virtual int Initialize(const ProcessingConfig& processing_config) = 0; |
| 517 | |
peah | 88ac853 | 2016-09-12 23:47:25 | [diff] [blame] | 518 | // TODO(peah): This method is a temporary solution used to take control |
| 519 | // over the parameters in the audio processing module and is likely to change. |
| 520 | virtual void ApplyConfig(const Config& config) = 0; |
| 521 | |
andrew@webrtc.org | ddbb8a2 | 2014-04-22 21:00:04 | [diff] [blame] | 522 | // TODO(ajm): Only intended for internal use. Make private and friend the |
| 523 | // necessary classes? |
| 524 | virtual int proc_sample_rate_hz() const = 0; |
| 525 | virtual int proc_split_sample_rate_hz() const = 0; |
Peter Kasting | 6955870 | 2016-01-13 00:26:35 | [diff] [blame] | 526 | virtual size_t num_input_channels() const = 0; |
| 527 | virtual size_t num_proc_channels() const = 0; |
| 528 | virtual size_t num_output_channels() const = 0; |
| 529 | virtual size_t num_reverse_channels() const = 0; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 530 | |
andrew@webrtc.org | 17342e5 | 2014-02-12 22:28:31 | [diff] [blame] | 531 | // Set to true when the output of AudioProcessing will be muted or in some |
| 532 | // other way not used. Ideally, the captured audio would still be processed, |
| 533 | // but some components may change behavior based on this information. |
Per Åhgren | 0a144a7 | 2021-02-09 07:47:51 | [diff] [blame] | 534 | // Default false. This method takes a lock. To achieve this in a lock-less |
| 535 | // manner the PostRuntimeSetting can instead be used. |
andrew@webrtc.org | 17342e5 | 2014-02-12 22:28:31 | [diff] [blame] | 536 | virtual void set_output_will_be_muted(bool muted) = 0; |
andrew@webrtc.org | 17342e5 | 2014-02-12 22:28:31 | [diff] [blame] | 537 | |
Per Åhgren | 0a144a7 | 2021-02-09 07:47:51 | [diff] [blame] | 538 | // Enqueues a runtime setting. |
Alessio Bazzica | c054e78 | 2018-04-16 10:10:09 | [diff] [blame] | 539 | virtual void SetRuntimeSetting(RuntimeSetting setting) = 0; |
| 540 | |
Per Åhgren | 0a144a7 | 2021-02-09 07:47:51 | [diff] [blame] | 541 | // Enqueues a runtime setting. Returns a bool indicating whether the |
| 542 | // enqueueing was successfull. |
Per Åhgren | 8eea117 | 2021-02-09 22:15:07 | [diff] [blame] | 543 | virtual bool PostRuntimeSetting(RuntimeSetting setting) = 0; |
Per Åhgren | 0a144a7 | 2021-02-09 07:47:51 | [diff] [blame] | 544 | |
Sam Zackrisson | 3bd444f | 2022-08-03 12:37:00 | [diff] [blame] | 545 | // Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio as |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 546 | // specified in `input_config` and `output_config`. `src` and `dest` may use |
Per Åhgren | 645f24c | 2020-03-16 11:06:02 | [diff] [blame] | 547 | // the same memory, if desired. |
| 548 | virtual int ProcessStream(const int16_t* const src, |
| 549 | const StreamConfig& input_config, |
| 550 | const StreamConfig& output_config, |
Per Åhgren | dc5522b | 2020-03-19 13:55:58 | [diff] [blame] | 551 | int16_t* const dest) = 0; |
Per Åhgren | 645f24c | 2020-03-16 11:06:02 | [diff] [blame] | 552 | |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 553 | // Accepts deinterleaved float audio with the range [-1, 1]. Each element of |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 554 | // `src` points to a channel buffer, arranged according to `input_stream`. At |
| 555 | // output, the channels will be arranged according to `output_stream` in |
| 556 | // `dest`. |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 557 | // |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 558 | // The output must have one channel or as many channels as the input. `src` |
| 559 | // and `dest` may use the same memory, if desired. |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 560 | virtual int ProcessStream(const float* const* src, |
| 561 | const StreamConfig& input_config, |
| 562 | const StreamConfig& output_config, |
| 563 | float* const* dest) = 0; |
| 564 | |
Sam Zackrisson | 3bd444f | 2022-08-03 12:37:00 | [diff] [blame] | 565 | // Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio for |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 566 | // the reverse direction audio stream as specified in `input_config` and |
| 567 | // `output_config`. `src` and `dest` may use the same memory, if desired. |
Per Åhgren | 645f24c | 2020-03-16 11:06:02 | [diff] [blame] | 568 | virtual int ProcessReverseStream(const int16_t* const src, |
| 569 | const StreamConfig& input_config, |
| 570 | const StreamConfig& output_config, |
| 571 | int16_t* const dest) = 0; |
| 572 | |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 573 | // Accepts deinterleaved float audio with the range [-1, 1]. Each element of |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 574 | // `data` points to a channel buffer, arranged according to `reverse_config`. |
ekmeyerson | 60d9b33 | 2015-08-14 17:35:55 | [diff] [blame] | 575 | virtual int ProcessReverseStream(const float* const* src, |
peah | de65ddc | 2016-09-16 22:02:15 | [diff] [blame] | 576 | const StreamConfig& input_config, |
| 577 | const StreamConfig& output_config, |
ekmeyerson | 60d9b33 | 2015-08-14 17:35:55 | [diff] [blame] | 578 | float* const* dest) = 0; |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 579 | |
Gustaf Ullberg | cb30726 | 2019-10-29 08:30:44 | [diff] [blame] | 580 | // Accepts deinterleaved float audio with the range [-1, 1]. Each element |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 581 | // of `data` points to a channel buffer, arranged according to |
| 582 | // `reverse_config`. |
Gustaf Ullberg | cb30726 | 2019-10-29 08:30:44 | [diff] [blame] | 583 | virtual int AnalyzeReverseStream(const float* const* data, |
| 584 | const StreamConfig& reverse_config) = 0; |
| 585 | |
Sam Zackrisson | 3bd444f | 2022-08-03 12:37:00 | [diff] [blame] | 586 | // Returns the most recently produced ~10 ms of the linear AEC output at a |
| 587 | // rate of 16 kHz. If there is more than one capture channel, a mono |
| 588 | // representation of the input is returned. Returns true/false to indicate |
| 589 | // whether an output returned. |
Per Åhgren | c20a19c | 2019-11-13 10:12:29 | [diff] [blame] | 590 | virtual bool GetLinearAecOutput( |
| 591 | rtc::ArrayView<std::array<float, 160>> linear_output) const = 0; |
| 592 | |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 593 | // This must be called prior to ProcessStream() if and only if adaptive analog |
| 594 | // gain control is enabled, to pass the current analog level from the audio |
Hanna Silen | cd59704 | 2021-11-02 10:02:48 | [diff] [blame] | 595 | // HAL. Must be within the range [0, 255]. |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 596 | virtual void set_stream_analog_level(int level) = 0; |
| 597 | |
Alessio Bazzica | fcf1af3 | 2022-09-07 15:14:26 | [diff] [blame] | 598 | // When an analog mode is set, this should be called after |
| 599 | // `set_stream_analog_level()` and `ProcessStream()` to obtain the recommended |
| 600 | // new analog level for the audio HAL. It is the user's responsibility to |
| 601 | // apply this level. |
Sam Zackrisson | f0d1c03 | 2019-03-27 12:28:08 | [diff] [blame] | 602 | virtual int recommended_stream_analog_level() const = 0; |
| 603 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 604 | // This must be called if and only if echo processing is enabled. |
| 605 | // |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 606 | // Sets the `delay` in ms between ProcessReverseStream() receiving a far-end |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 607 | // frame and ProcessStream() receiving a near-end frame containing the |
| 608 | // corresponding echo. On the client-side this can be expressed as |
| 609 | // delay = (t_render - t_analyze) + (t_process - t_capture) |
| 610 | // where, |
aluebs | b031955 | 2016-03-18 03:39:53 | [diff] [blame] | 611 | // - t_analyze is the time a frame is passed to ProcessReverseStream() and |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 612 | // t_render is the time the first sample of the same frame is rendered by |
| 613 | // the audio hardware. |
| 614 | // - t_capture is the time the first sample of a frame is captured by the |
alessiob | 13fc180 | 2017-04-19 12:35:51 | [diff] [blame] | 615 | // audio hardware and t_process is the time the same frame is passed to |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 616 | // ProcessStream(). |
| 617 | virtual int set_stream_delay_ms(int delay) = 0; |
| 618 | virtual int stream_delay_ms() const = 0; |
| 619 | |
andrew@webrtc.org | 75dd288 | 2014-02-11 20:52:30 | [diff] [blame] | 620 | // Call to signal that a key press occurred (true) or did not occur (false) |
| 621 | // with this chunk of audio. |
| 622 | virtual void set_stream_key_pressed(bool key_pressed) = 0; |
andrew@webrtc.org | 75dd288 | 2014-02-11 20:52:30 | [diff] [blame] | 623 | |
Per Åhgren | 09e9a83 | 2020-05-11 09:03:47 | [diff] [blame] | 624 | // Creates and attaches an webrtc::AecDump for recording debugging |
| 625 | // information. |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 626 | // The `worker_queue` may not be null and must outlive the created |
Per Åhgren | 09e9a83 | 2020-05-11 09:03:47 | [diff] [blame] | 627 | // AecDump instance. |max_log_size_bytes == -1| means the log size |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 628 | // will be unlimited. `handle` may not be null. The AecDump takes |
| 629 | // responsibility for `handle` and closes it in the destructor. A |
Per Åhgren | 09e9a83 | 2020-05-11 09:03:47 | [diff] [blame] | 630 | // return value of true indicates that the file has been |
| 631 | // sucessfully opened, while a value of false indicates that |
| 632 | // opening the file failed. |
Danil Chapovalov | 1ecf29c | 2024-01-09 10:52:10 | [diff] [blame] | 633 | virtual bool CreateAndAttachAecDump( |
| 634 | absl::string_view file_name, |
| 635 | int64_t max_log_size_bytes, |
Danil Chapovalov | e052eee | 2024-01-15 10:42:13 | [diff] [blame] | 636 | absl::Nonnull<TaskQueueBase*> worker_queue) = 0; |
Danil Chapovalov | 1ecf29c | 2024-01-09 10:52:10 | [diff] [blame] | 637 | virtual bool CreateAndAttachAecDump( |
| 638 | absl::Nonnull<FILE*> handle, |
| 639 | int64_t max_log_size_bytes, |
Danil Chapovalov | e052eee | 2024-01-15 10:42:13 | [diff] [blame] | 640 | absl::Nonnull<TaskQueueBase*> worker_queue) = 0; |
Per Åhgren | 09e9a83 | 2020-05-11 09:03:47 | [diff] [blame] | 641 | |
| 642 | // TODO(webrtc:5298) Deprecated variant. |
aleloi | 868f32f | 2017-05-23 14:20:05 | [diff] [blame] | 643 | // Attaches provided webrtc::AecDump for recording debugging |
| 644 | // information. Log file and maximum file size logic is supposed to |
| 645 | // be handled by implementing instance of AecDump. Calling this |
| 646 | // method when another AecDump is attached resets the active AecDump |
| 647 | // with a new one. This causes the d-tor of the earlier AecDump to |
| 648 | // be called. The d-tor call may block until all pending logging |
| 649 | // tasks are completed. |
Alex Loiko | be767e0 | 2017-06-08 07:45:03 | [diff] [blame] | 650 | virtual void AttachAecDump(std::unique_ptr<AecDump> aec_dump) = 0; |
aleloi | 868f32f | 2017-05-23 14:20:05 | [diff] [blame] | 651 | |
| 652 | // If no AecDump is attached, this has no effect. If an AecDump is |
| 653 | // attached, it's destructor is called. The d-tor may block until |
| 654 | // all pending logging tasks are completed. |
Alex Loiko | be767e0 | 2017-06-08 07:45:03 | [diff] [blame] | 655 | virtual void DetachAecDump() = 0; |
aleloi | 868f32f | 2017-05-23 14:20:05 | [diff] [blame] | 656 | |
Per Åhgren | cf4c872 | 2019-12-30 13:32:14 | [diff] [blame] | 657 | // Get audio processing statistics. |
| 658 | virtual AudioProcessingStats GetStatistics() = 0; |
Artem Titov | 0b48930 | 2021-07-28 18:50:03 | [diff] [blame] | 659 | // TODO(webrtc:5298) Deprecated variant. The `has_remote_tracks` argument |
Per Åhgren | cf4c872 | 2019-12-30 13:32:14 | [diff] [blame] | 660 | // should be set if there are active remote tracks (this would usually be true |
| 661 | // during a call). If there are no remote tracks some of the stats will not be |
| 662 | // set by AudioProcessing, because they only make sense if there is at least |
| 663 | // one remote track. |
| 664 | virtual AudioProcessingStats GetStatistics(bool has_remote_tracks) = 0; |
Ivo Creusen | ae026096 | 2017-11-20 12:07:16 | [diff] [blame] | 665 | |
henrik.lundin | adf0635 | 2017-04-05 12:48:24 | [diff] [blame] | 666 | // Returns the last applied configuration. |
henrik.lundin | 7749286 | 2017-04-07 06:28:09 | [diff] [blame] | 667 | virtual AudioProcessing::Config GetConfig() const = 0; |
henrik.lundin | adf0635 | 2017-04-05 12:48:24 | [diff] [blame] | 668 | |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 | [diff] [blame] | 669 | enum Error { |
| 670 | // Fatal errors. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 671 | kNoError = 0, |
| 672 | kUnspecifiedError = -1, |
| 673 | kCreationFailedError = -2, |
| 674 | kUnsupportedComponentError = -3, |
| 675 | kUnsupportedFunctionError = -4, |
| 676 | kNullPointerError = -5, |
| 677 | kBadParameterError = -6, |
| 678 | kBadSampleRateError = -7, |
| 679 | kBadDataLengthError = -8, |
| 680 | kBadNumberChannelsError = -9, |
| 681 | kFileError = -10, |
| 682 | kStreamParameterNotSetError = -11, |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 | [diff] [blame] | 683 | kNotEnabledError = -12, |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 684 | |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 | [diff] [blame] | 685 | // Warnings are non-fatal. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 686 | // This results when a set_stream_ parameter is out of range. Processing |
| 687 | // will continue, but the parameter may have been truncated. |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 | [diff] [blame] | 688 | kBadStreamParameterWarning = -13 |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 689 | }; |
andrew@webrtc.org | 56e4a05 | 2014-02-27 22:23:17 | [diff] [blame] | 690 | |
Per Åhgren | 2507f8c | 2020-03-19 11:33:29 | [diff] [blame] | 691 | // Native rates supported by the integer interfaces. |
andrew@webrtc.org | ddbb8a2 | 2014-04-22 21:00:04 | [diff] [blame] | 692 | enum NativeRate { |
andrew@webrtc.org | 56e4a05 | 2014-02-27 22:23:17 | [diff] [blame] | 693 | kSampleRate8kHz = 8000, |
| 694 | kSampleRate16kHz = 16000, |
aluebs@webrtc.org | 087da13 | 2014-11-17 23:01:23 | [diff] [blame] | 695 | kSampleRate32kHz = 32000, |
| 696 | kSampleRate48kHz = 48000 |
andrew@webrtc.org | 56e4a05 | 2014-02-27 22:23:17 | [diff] [blame] | 697 | }; |
andrew@webrtc.org | ddbb8a2 | 2014-04-22 21:00:04 | [diff] [blame] | 698 | |
kwiberg | d59d3bb | 2016-09-13 14:49:33 | [diff] [blame] | 699 | // TODO(kwiberg): We currently need to support a compiler (Visual C++) that |
| 700 | // complains if we don't explicitly state the size of the array here. Remove |
| 701 | // the size when that's no longer the case. |
| 702 | static constexpr int kNativeSampleRatesHz[4] = { |
| 703 | kSampleRate8kHz, kSampleRate16kHz, kSampleRate32kHz, kSampleRate48kHz}; |
| 704 | static constexpr size_t kNumNativeSampleRates = |
| 705 | arraysize(kNativeSampleRatesHz); |
| 706 | static constexpr int kMaxNativeSampleRateHz = |
| 707 | kNativeSampleRatesHz[kNumNativeSampleRates - 1]; |
Alejandro Luebs | cdfe20b | 2015-09-23 19:49:12 | [diff] [blame] | 708 | |
Sam Zackrisson | 3bd444f | 2022-08-03 12:37:00 | [diff] [blame] | 709 | // APM processes audio in chunks of about 10 ms. See GetFrameSize() for |
| 710 | // details. |
Per Åhgren | 12dc274 | 2020-12-08 08:40:35 | [diff] [blame] | 711 | static constexpr int kChunkSizeMs = 10; |
Sam Zackrisson | 3bd444f | 2022-08-03 12:37:00 | [diff] [blame] | 712 | |
| 713 | // Returns floor(sample_rate_hz/100): the number of samples per channel used |
| 714 | // as input and output to the audio processing module in calls to |
| 715 | // ProcessStream, ProcessReverseStream, AnalyzeReverseStream, and |
| 716 | // GetLinearAecOutput. |
| 717 | // |
| 718 | // This is exactly 10 ms for sample rates divisible by 100. For example: |
| 719 | // - 48000 Hz (480 samples per channel), |
| 720 | // - 44100 Hz (441 samples per channel), |
| 721 | // - 16000 Hz (160 samples per channel). |
| 722 | // |
| 723 | // Sample rates not divisible by 100 are received/produced in frames of |
| 724 | // approximately 10 ms. For example: |
| 725 | // - 22050 Hz (220 samples per channel, or ~9.98 ms per frame), |
| 726 | // - 11025 Hz (110 samples per channel, or ~9.98 ms per frame). |
| 727 | // These nondivisible sample rates yield lower audio quality compared to |
| 728 | // multiples of 100. Internal resampling to 10 ms frames causes a simulated |
| 729 | // clock drift effect which impacts the performance of (for example) echo |
| 730 | // cancellation. |
| 731 | static int GetFrameSize(int sample_rate_hz) { return sample_rate_hz / 100; } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 732 | }; |
| 733 | |
Mirko Bonadei | 3d25530 | 2018-10-11 08:50:45 | [diff] [blame] | 734 | class RTC_EXPORT AudioProcessingBuilder { |
Ivo Creusen | 5ec7e12 | 2017-12-22 10:35:59 | [diff] [blame] | 735 | public: |
| 736 | AudioProcessingBuilder(); |
Alessio Bazzica | 20a9ac6 | 2021-10-14 08:55:08 | [diff] [blame] | 737 | AudioProcessingBuilder(const AudioProcessingBuilder&) = delete; |
| 738 | AudioProcessingBuilder& operator=(const AudioProcessingBuilder&) = delete; |
Ivo Creusen | 5ec7e12 | 2017-12-22 10:35:59 | [diff] [blame] | 739 | ~AudioProcessingBuilder(); |
Alessio Bazzica | 20a9ac6 | 2021-10-14 08:55:08 | [diff] [blame] | 740 | |
| 741 | // Sets the APM configuration. |
| 742 | AudioProcessingBuilder& SetConfig(const AudioProcessing::Config& config) { |
| 743 | config_ = config; |
| 744 | return *this; |
| 745 | } |
| 746 | |
| 747 | // Sets the echo controller factory to inject when APM is created. |
Ivo Creusen | 5ec7e12 | 2017-12-22 10:35:59 | [diff] [blame] | 748 | AudioProcessingBuilder& SetEchoControlFactory( |
Per Åhgren | cc73ed3 | 2020-04-26 21:56:17 | [diff] [blame] | 749 | std::unique_ptr<EchoControlFactory> echo_control_factory) { |
| 750 | echo_control_factory_ = std::move(echo_control_factory); |
| 751 | return *this; |
| 752 | } |
Alessio Bazzica | 20a9ac6 | 2021-10-14 08:55:08 | [diff] [blame] | 753 | |
| 754 | // Sets the capture post-processing sub-module to inject when APM is created. |
Ivo Creusen | 5ec7e12 | 2017-12-22 10:35:59 | [diff] [blame] | 755 | AudioProcessingBuilder& SetCapturePostProcessing( |
Per Åhgren | cc73ed3 | 2020-04-26 21:56:17 | [diff] [blame] | 756 | std::unique_ptr<CustomProcessing> capture_post_processing) { |
| 757 | capture_post_processing_ = std::move(capture_post_processing); |
| 758 | return *this; |
| 759 | } |
Alessio Bazzica | 20a9ac6 | 2021-10-14 08:55:08 | [diff] [blame] | 760 | |
| 761 | // Sets the render pre-processing sub-module to inject when APM is created. |
Ivo Creusen | 5ec7e12 | 2017-12-22 10:35:59 | [diff] [blame] | 762 | AudioProcessingBuilder& SetRenderPreProcessing( |
Per Åhgren | cc73ed3 | 2020-04-26 21:56:17 | [diff] [blame] | 763 | std::unique_ptr<CustomProcessing> render_pre_processing) { |
| 764 | render_pre_processing_ = std::move(render_pre_processing); |
| 765 | return *this; |
| 766 | } |
Alessio Bazzica | 20a9ac6 | 2021-10-14 08:55:08 | [diff] [blame] | 767 | |
| 768 | // Sets the echo detector to inject when APM is created. |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 769 | AudioProcessingBuilder& SetEchoDetector( |
Per Åhgren | cc73ed3 | 2020-04-26 21:56:17 | [diff] [blame] | 770 | rtc::scoped_refptr<EchoDetector> echo_detector) { |
| 771 | echo_detector_ = std::move(echo_detector); |
| 772 | return *this; |
| 773 | } |
Alessio Bazzica | 20a9ac6 | 2021-10-14 08:55:08 | [diff] [blame] | 774 | |
| 775 | // Sets the capture analyzer sub-module to inject when APM is created. |
Valeriia Nemychnikova | f06eb57 | 2018-08-29 08:37:09 | [diff] [blame] | 776 | AudioProcessingBuilder& SetCaptureAnalyzer( |
Per Åhgren | cc73ed3 | 2020-04-26 21:56:17 | [diff] [blame] | 777 | std::unique_ptr<CustomAudioAnalyzer> capture_analyzer) { |
| 778 | capture_analyzer_ = std::move(capture_analyzer); |
| 779 | return *this; |
| 780 | } |
Alessio Bazzica | 20a9ac6 | 2021-10-14 08:55:08 | [diff] [blame] | 781 | |
| 782 | // Creates an APM instance with the specified config or the default one if |
| 783 | // unspecified. Injects the specified components transferring the ownership |
| 784 | // to the newly created APM instance - i.e., except for the config, the |
| 785 | // builder is reset to its initial state. |
Niels Möller | 4f776ac | 2021-07-02 09:30:54 | [diff] [blame] | 786 | rtc::scoped_refptr<AudioProcessing> Create(); |
Ivo Creusen | 5ec7e12 | 2017-12-22 10:35:59 | [diff] [blame] | 787 | |
| 788 | private: |
Alessio Bazzica | 20a9ac6 | 2021-10-14 08:55:08 | [diff] [blame] | 789 | AudioProcessing::Config config_; |
Ivo Creusen | 5ec7e12 | 2017-12-22 10:35:59 | [diff] [blame] | 790 | std::unique_ptr<EchoControlFactory> echo_control_factory_; |
| 791 | std::unique_ptr<CustomProcessing> capture_post_processing_; |
| 792 | std::unique_ptr<CustomProcessing> render_pre_processing_; |
Ivo Creusen | d1f970d | 2018-06-14 09:02:03 | [diff] [blame] | 793 | rtc::scoped_refptr<EchoDetector> echo_detector_; |
Valeriia Nemychnikova | f06eb57 | 2018-08-29 08:37:09 | [diff] [blame] | 794 | std::unique_ptr<CustomAudioAnalyzer> capture_analyzer_; |
Ivo Creusen | 5ec7e12 | 2017-12-22 10:35:59 | [diff] [blame] | 795 | }; |
| 796 | |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 797 | class StreamConfig { |
| 798 | public: |
| 799 | // sample_rate_hz: The sampling rate of the stream. |
Henrik Lundin | 64253a9 | 2022-02-04 09:02:48 | [diff] [blame] | 800 | // num_channels: The number of audio channels in the stream. |
Alessio Bazzica | c7d0e42 | 2022-02-04 16:06:55 | [diff] [blame] | 801 | StreamConfig(int sample_rate_hz = 0, size_t num_channels = 0) |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 802 | : sample_rate_hz_(sample_rate_hz), |
| 803 | num_channels_(num_channels), |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 804 | num_frames_(calculate_frames(sample_rate_hz)) {} |
| 805 | |
| 806 | void set_sample_rate_hz(int value) { |
| 807 | sample_rate_hz_ = value; |
| 808 | num_frames_ = calculate_frames(value); |
| 809 | } |
Peter Kasting | 6955870 | 2016-01-13 00:26:35 | [diff] [blame] | 810 | void set_num_channels(size_t value) { num_channels_ = value; } |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 811 | |
| 812 | int sample_rate_hz() const { return sample_rate_hz_; } |
| 813 | |
Henrik Lundin | 64253a9 | 2022-02-04 09:02:48 | [diff] [blame] | 814 | // The number of channels in the stream. |
Peter Kasting | 6955870 | 2016-01-13 00:26:35 | [diff] [blame] | 815 | size_t num_channels() const { return num_channels_; } |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 816 | |
Peter Kasting | dce40cf | 2015-08-24 21:52:23 | [diff] [blame] | 817 | size_t num_frames() const { return num_frames_; } |
| 818 | size_t num_samples() const { return num_channels_ * num_frames_; } |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 819 | |
| 820 | bool operator==(const StreamConfig& other) const { |
| 821 | return sample_rate_hz_ == other.sample_rate_hz_ && |
Henrik Lundin | 64253a9 | 2022-02-04 09:02:48 | [diff] [blame] | 822 | num_channels_ == other.num_channels_; |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 823 | } |
| 824 | |
| 825 | bool operator!=(const StreamConfig& other) const { return !(*this == other); } |
| 826 | |
| 827 | private: |
Peter Kasting | dce40cf | 2015-08-24 21:52:23 | [diff] [blame] | 828 | static size_t calculate_frames(int sample_rate_hz) { |
Sam Zackrisson | 3bd444f | 2022-08-03 12:37:00 | [diff] [blame] | 829 | return static_cast<size_t>(AudioProcessing::GetFrameSize(sample_rate_hz)); |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 830 | } |
| 831 | |
| 832 | int sample_rate_hz_; |
Peter Kasting | 6955870 | 2016-01-13 00:26:35 | [diff] [blame] | 833 | size_t num_channels_; |
Peter Kasting | dce40cf | 2015-08-24 21:52:23 | [diff] [blame] | 834 | size_t num_frames_; |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 835 | }; |
| 836 | |
| 837 | class ProcessingConfig { |
| 838 | public: |
| 839 | enum StreamName { |
| 840 | kInputStream, |
| 841 | kOutputStream, |
ekmeyerson | 60d9b33 | 2015-08-14 17:35:55 | [diff] [blame] | 842 | kReverseInputStream, |
| 843 | kReverseOutputStream, |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 844 | kNumStreamNames, |
| 845 | }; |
| 846 | |
| 847 | const StreamConfig& input_stream() const { |
| 848 | return streams[StreamName::kInputStream]; |
| 849 | } |
| 850 | const StreamConfig& output_stream() const { |
| 851 | return streams[StreamName::kOutputStream]; |
| 852 | } |
ekmeyerson | 60d9b33 | 2015-08-14 17:35:55 | [diff] [blame] | 853 | const StreamConfig& reverse_input_stream() const { |
| 854 | return streams[StreamName::kReverseInputStream]; |
| 855 | } |
| 856 | const StreamConfig& reverse_output_stream() const { |
| 857 | return streams[StreamName::kReverseOutputStream]; |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 858 | } |
| 859 | |
| 860 | StreamConfig& input_stream() { return streams[StreamName::kInputStream]; } |
| 861 | StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; } |
ekmeyerson | 60d9b33 | 2015-08-14 17:35:55 | [diff] [blame] | 862 | StreamConfig& reverse_input_stream() { |
| 863 | return streams[StreamName::kReverseInputStream]; |
| 864 | } |
| 865 | StreamConfig& reverse_output_stream() { |
| 866 | return streams[StreamName::kReverseOutputStream]; |
| 867 | } |
Michael Graczyk | 86c6d33 | 2015-07-23 18:41:39 | [diff] [blame] | 868 | |
| 869 | bool operator==(const ProcessingConfig& other) const { |
| 870 | for (int i = 0; i < StreamName::kNumStreamNames; ++i) { |
| 871 | if (this->streams[i] != other.streams[i]) { |
| 872 | return false; |
| 873 | } |
| 874 | } |
| 875 | return true; |
| 876 | } |
| 877 | |
| 878 | bool operator!=(const ProcessingConfig& other) const { |
| 879 | return !(*this == other); |
| 880 | } |
| 881 | |
| 882 | StreamConfig streams[StreamName::kNumStreamNames]; |
| 883 | }; |
| 884 | |
Valeriia Nemychnikova | f06eb57 | 2018-08-29 08:37:09 | [diff] [blame] | 885 | // Experimental interface for a custom analysis submodule. |
| 886 | class CustomAudioAnalyzer { |
| 887 | public: |
| 888 | // (Re-) Initializes the submodule. |
| 889 | virtual void Initialize(int sample_rate_hz, int num_channels) = 0; |
| 890 | // Analyzes the given capture or render signal. |
| 891 | virtual void Analyze(const AudioBuffer* audio) = 0; |
| 892 | // Returns a string representation of the module state. |
| 893 | virtual std::string ToString() const = 0; |
| 894 | |
| 895 | virtual ~CustomAudioAnalyzer() {} |
| 896 | }; |
| 897 | |
Alex Loiko | 5825aa6 | 2017-12-18 15:02:40 | [diff] [blame] | 898 | // Interface for a custom processing submodule. |
| 899 | class CustomProcessing { |
Sam Zackrisson | 0beac58 | 2017-09-25 10:04:02 | [diff] [blame] | 900 | public: |
| 901 | // (Re-)Initializes the submodule. |
| 902 | virtual void Initialize(int sample_rate_hz, int num_channels) = 0; |
| 903 | // Processes the given capture or render signal. |
| 904 | virtual void Process(AudioBuffer* audio) = 0; |
| 905 | // Returns a string representation of the module state. |
| 906 | virtual std::string ToString() const = 0; |
Alex Loiko | 73ec019 | 2018-05-15 08:52:28 | [diff] [blame] | 907 | // Handles RuntimeSettings. TODO(webrtc:9262): make pure virtual |
| 908 | // after updating dependencies. |
| 909 | virtual void SetRuntimeSetting(AudioProcessing::RuntimeSetting setting); |
Sam Zackrisson | 0beac58 | 2017-09-25 10:04:02 | [diff] [blame] | 910 | |
Alex Loiko | 5825aa6 | 2017-12-18 15:02:40 | [diff] [blame] | 911 | virtual ~CustomProcessing() {} |
Sam Zackrisson | 0beac58 | 2017-09-25 10:04:02 | [diff] [blame] | 912 | }; |
| 913 | |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 914 | // Interface for an echo detector submodule. |
Harald Alvestrand | 78f905e | 2023-11-02 14:09:26 | [diff] [blame] | 915 | class EchoDetector : public RefCountInterface { |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 916 | public: |
| 917 | // (Re-)Initializes the submodule. |
Ivo Creusen | 647ef09 | 2018-03-14 16:13:48 | [diff] [blame] | 918 | virtual void Initialize(int capture_sample_rate_hz, |
| 919 | int num_capture_channels, |
| 920 | int render_sample_rate_hz, |
| 921 | int num_render_channels) = 0; |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 922 | |
Sam Zackrisson | 03cb7e5 | 2021-12-06 14:40:04 | [diff] [blame] | 923 | // Analysis (not changing) of the first channel of the render signal. |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 924 | virtual void AnalyzeRenderAudio(rtc::ArrayView<const float> render_audio) = 0; |
| 925 | |
| 926 | // Analysis (not changing) of the capture signal. |
| 927 | virtual void AnalyzeCaptureAudio( |
| 928 | rtc::ArrayView<const float> capture_audio) = 0; |
| 929 | |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 930 | struct Metrics { |
Ivo Creusen | bb826c9 | 2020-04-29 12:34:48 | [diff] [blame] | 931 | absl::optional<double> echo_likelihood; |
| 932 | absl::optional<double> echo_likelihood_recent_max; |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 933 | }; |
| 934 | |
| 935 | // Collect current metrics from the echo detector. |
| 936 | virtual Metrics GetMetrics() const = 0; |
Ivo Creusen | 09fa4b0 | 2018-01-11 15:08:54 | [diff] [blame] | 937 | }; |
| 938 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 939 | } // namespace webrtc |
| 940 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 941 | #endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |