| /* |
| * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |
| #define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |
| |
| // MSVC++ requires this to be set before any other includes to get M_PI. |
| #ifndef _USE_MATH_DEFINES |
| #define _USE_MATH_DEFINES |
| #endif |
| |
| #include <math.h> |
| #include <stddef.h> // size_t |
| #include <stdio.h> // FILE |
| #include <string.h> |
| #include <vector> |
| |
| #include "api/audio/echo_canceller3_config.h" |
| #include "api/audio/echo_control.h" |
| #include "api/optional.h" |
| #include "modules/audio_processing/beamformer/array_util.h" |
| #include "modules/audio_processing/include/audio_processing_statistics.h" |
| #include "modules/audio_processing/include/config.h" |
| #include "rtc_base/arraysize.h" |
| #include "rtc_base/deprecation.h" |
| #include "rtc_base/platform_file.h" |
| #include "rtc_base/refcount.h" |
| #include "rtc_base/scoped_ref_ptr.h" |
| #include "typedefs.h" // NOLINT(build/include) |
| |
| namespace webrtc { |
| |
| struct AecCore; |
| |
| class AecDump; |
| class AudioBuffer; |
| class AudioFrame; |
| |
| class NonlinearBeamformer; |
| |
| class StreamConfig; |
| class ProcessingConfig; |
| |
| class EchoCancellation; |
| class EchoControlMobile; |
| class EchoDetector; |
| class GainControl; |
| class HighPassFilter; |
| class LevelEstimator; |
| class NoiseSuppression; |
| class CustomProcessing; |
| class VoiceDetection; |
| |
| // webrtc:8665, addedd temporarily to avoid breaking dependencies. |
| typedef CustomProcessing PostProcessing; |
| |
| // Use to enable the extended filter mode in the AEC, along with robustness |
| // measures around the reported system delays. It comes with a significant |
| // increase in AEC complexity, but is much more robust to unreliable reported |
| // delays. |
| // |
| // Detailed changes to the algorithm: |
| // - The filter length is changed from 48 to 128 ms. This comes with tuning of |
| // several parameters: i) filter adaptation stepsize and error threshold; |
| // ii) non-linear processing smoothing and overdrive. |
| // - Option to ignore the reported delays on platforms which we deem |
| // sufficiently unreliable. See WEBRTC_UNTRUSTED_DELAY in echo_cancellation.c. |
| // - Faster startup times by removing the excessive "startup phase" processing |
| // of reported delays. |
| // - Much more conservative adjustments to the far-end read pointer. We smooth |
| // the delay difference more heavily, and back off from the difference more. |
| // Adjustments force a readaptation of the filter, so they should be avoided |
| // except when really necessary. |
| struct ExtendedFilter { |
| ExtendedFilter() : enabled(false) {} |
| explicit ExtendedFilter(bool enabled) : enabled(enabled) {} |
| static const ConfigOptionID identifier = ConfigOptionID::kExtendedFilter; |
| bool enabled; |
| }; |
| |
| // Enables the refined linear filter adaptation in the echo canceller. |
| // This configuration only applies to EchoCancellation and not |
| // EchoControlMobile. It can be set in the constructor |
| // or using AudioProcessing::SetExtraOptions(). |
| struct RefinedAdaptiveFilter { |
| RefinedAdaptiveFilter() : enabled(false) {} |
| explicit RefinedAdaptiveFilter(bool enabled) : enabled(enabled) {} |
| static const ConfigOptionID identifier = |
| ConfigOptionID::kAecRefinedAdaptiveFilter; |
| bool enabled; |
| }; |
| |
| // Enables delay-agnostic echo cancellation. This feature relies on internally |
| // estimated delays between the process and reverse streams, thus not relying |
| // on reported system delays. This configuration only applies to |
| // EchoCancellation and not EchoControlMobile. It can be set in the constructor |
| // or using AudioProcessing::SetExtraOptions(). |
| struct DelayAgnostic { |
| DelayAgnostic() : enabled(false) {} |
| explicit DelayAgnostic(bool enabled) : enabled(enabled) {} |
| static const ConfigOptionID identifier = ConfigOptionID::kDelayAgnostic; |
| bool enabled; |
| }; |
| |
| // Use to enable experimental gain control (AGC). At startup the experimental |
| // AGC moves the microphone volume up to |startup_min_volume| if the current |
| // microphone volume is set too low. The value is clamped to its operating range |
| // [12, 255]. Here, 255 maps to 100%. |
| // |
| // Must be provided through AudioProcessingBuilder().Create(config). |
| #if defined(WEBRTC_CHROMIUM_BUILD) |
| static const int kAgcStartupMinVolume = 85; |
| #else |
| static const int kAgcStartupMinVolume = 0; |
| #endif // defined(WEBRTC_CHROMIUM_BUILD) |
| static constexpr int kClippedLevelMin = 70; |
| struct ExperimentalAgc { |
| ExperimentalAgc() = default; |
| explicit ExperimentalAgc(bool enabled) : enabled(enabled) {} |
| ExperimentalAgc(bool enabled, int startup_min_volume) |
| : enabled(enabled), startup_min_volume(startup_min_volume) {} |
| ExperimentalAgc(bool enabled, int startup_min_volume, int clipped_level_min) |
| : enabled(enabled), |
| startup_min_volume(startup_min_volume), |
| clipped_level_min(clipped_level_min) {} |
| static const ConfigOptionID identifier = ConfigOptionID::kExperimentalAgc; |
| bool enabled = true; |
| int startup_min_volume = kAgcStartupMinVolume; |
| // Lowest microphone level that will be applied in response to clipping. |
| int clipped_level_min = kClippedLevelMin; |
| }; |
| |
| // Use to enable experimental noise suppression. It can be set in the |
| // constructor or using AudioProcessing::SetExtraOptions(). |
| struct ExperimentalNs { |
| ExperimentalNs() : enabled(false) {} |
| explicit ExperimentalNs(bool enabled) : enabled(enabled) {} |
| static const ConfigOptionID identifier = ConfigOptionID::kExperimentalNs; |
| bool enabled; |
| }; |
| |
| // Use to enable beamforming. Must be provided through the constructor. It will |
| // have no impact if used with AudioProcessing::SetExtraOptions(). |
| struct Beamforming { |
| Beamforming(); |
| Beamforming(bool enabled, const std::vector<Point>& array_geometry); |
| Beamforming(bool enabled, |
| const std::vector<Point>& array_geometry, |
| SphericalPointf target_direction); |
| ~Beamforming(); |
| |
| static const ConfigOptionID identifier = ConfigOptionID::kBeamforming; |
| const bool enabled; |
| const std::vector<Point> array_geometry; |
| const SphericalPointf target_direction; |
| }; |
| |
| // Use to enable intelligibility enhancer in audio processing. |
| // |
| // Note: If enabled and the reverse stream has more than one output channel, |
| // the reverse stream will become an upmixed mono signal. |
| struct Intelligibility { |
| Intelligibility() : enabled(false) {} |
| explicit Intelligibility(bool enabled) : enabled(enabled) {} |
| static const ConfigOptionID identifier = ConfigOptionID::kIntelligibility; |
| bool enabled; |
| }; |
| |
| // The Audio Processing Module (APM) provides a collection of voice processing |
| // components designed for real-time communications software. |
| // |
| // APM operates on two audio streams on a frame-by-frame basis. Frames of the |
| // primary stream, on which all processing is applied, are passed to |
| // |ProcessStream()|. Frames of the reverse direction stream are passed to |
| // |ProcessReverseStream()|. On the client-side, this will typically be the |
| // near-end (capture) and far-end (render) streams, respectively. APM should be |
| // placed in the signal chain as close to the audio hardware abstraction layer |
| // (HAL) as possible. |
| // |
| // On the server-side, the reverse stream will normally not be used, with |
| // processing occurring on each incoming stream. |
| // |
| // Component interfaces follow a similar pattern and are accessed through |
| // corresponding getters in APM. All components are disabled at create-time, |
| // with default settings that are recommended for most situations. New settings |
| // can be applied without enabling a component. Enabling a component triggers |
| // memory allocation and initialization to allow it to start processing the |
| // streams. |
| // |
| // Thread safety is provided with the following assumptions to reduce locking |
| // overhead: |
| // 1. The stream getters and setters are called from the same thread as |
| // ProcessStream(). More precisely, stream functions are never called |
| // concurrently with ProcessStream(). |
| // 2. Parameter getters are never called concurrently with the corresponding |
| // setter. |
| // |
| // APM accepts only linear PCM audio data in chunks of 10 ms. The int16 |
| // interfaces use interleaved data, while the float interfaces use deinterleaved |
| // data. |
| // |
| // Usage example, omitting error checking: |
| // AudioProcessing* apm = AudioProcessingBuilder().Create(); |
| // |
| // AudioProcessing::Config config; |
| // config.level_controller.enabled = true; |
| // config.high_pass_filter.enabled = true; |
| // apm->ApplyConfig(config) |
| // |
| // apm->echo_cancellation()->enable_drift_compensation(false); |
| // apm->echo_cancellation()->Enable(true); |
| // |
| // apm->noise_reduction()->set_level(kHighSuppression); |
| // apm->noise_reduction()->Enable(true); |
| // |
| // apm->gain_control()->set_analog_level_limits(0, 255); |
| // apm->gain_control()->set_mode(kAdaptiveAnalog); |
| // apm->gain_control()->Enable(true); |
| // |
| // apm->voice_detection()->Enable(true); |
| // |
| // // Start a voice call... |
| // |
| // // ... Render frame arrives bound for the audio HAL ... |
| // apm->ProcessReverseStream(render_frame); |
| // |
| // // ... Capture frame arrives from the audio HAL ... |
| // // Call required set_stream_ functions. |
| // apm->set_stream_delay_ms(delay_ms); |
| // apm->gain_control()->set_stream_analog_level(analog_level); |
| // |
| // apm->ProcessStream(capture_frame); |
| // |
| // // Call required stream_ functions. |
| // analog_level = apm->gain_control()->stream_analog_level(); |
| // has_voice = apm->stream_has_voice(); |
| // |
| // // Repeate render and capture processing for the duration of the call... |
| // // Start a new call... |
| // apm->Initialize(); |
| // |
| // // Close the application... |
| // delete apm; |
| // |
| class AudioProcessing : public rtc::RefCountInterface { |
| public: |
| // The struct below constitutes the new parameter scheme for the audio |
| // processing. It is being introduced gradually and until it is fully |
| // introduced, it is prone to change. |
| // TODO(peah): Remove this comment once the new config scheme is fully rolled |
| // out. |
| // |
| // The parameters and behavior of the audio processing module are controlled |
| // by changing the default values in the AudioProcessing::Config struct. |
| // The config is applied by passing the struct to the ApplyConfig method. |
| struct Config { |
| struct LevelController { |
| bool enabled = false; |
| |
| // Sets the initial peak level to use inside the level controller in order |
| // to compute the signal gain. The unit for the peak level is dBFS and |
| // the allowed range is [-100, 0]. |
| float initial_peak_level_dbfs = -6.0206f; |
| } level_controller; |
| struct ResidualEchoDetector { |
| bool enabled = true; |
| } residual_echo_detector; |
| |
| struct HighPassFilter { |
| bool enabled = false; |
| } high_pass_filter; |
| |
| // Enables the next generation AGC functionality. This feature replaces the |
| // standard methods of gain control in the previous AGC. |
| // The functionality is not yet activated in the code and turning this on |
| // does not yet have the desired behavior. |
| struct GainController2 { |
| bool enabled = false; |
| float fixed_gain_db = 0.f; |
| } gain_controller2; |
| |
| // Explicit copy assignment implementation to avoid issues with memory |
| // sanitizer complaints in case of self-assignment. |
| // TODO(peah): Add buildflag to ensure that this is only included for memory |
| // sanitizer builds. |
| Config& operator=(const Config& config) { |
| if (this != &config) { |
| memcpy(this, &config, sizeof(*this)); |
| } |
| return *this; |
| } |
| }; |
| |
| // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone. |
| enum ChannelLayout { |
| kMono, |
| // Left, right. |
| kStereo, |
| // Mono, keyboard, and mic. |
| kMonoAndKeyboard, |
| // Left, right, keyboard, and mic. |
| kStereoAndKeyboard |
| }; |
| |
| ~AudioProcessing() override {} |
| |
| // Initializes internal states, while retaining all user settings. This |
| // should be called before beginning to process a new audio stream. However, |
| // it is not necessary to call before processing the first stream after |
| // creation. |
| // |
| // It is also not necessary to call if the audio parameters (sample |
| // rate and number of channels) have changed. Passing updated parameters |
| // directly to |ProcessStream()| and |ProcessReverseStream()| is permissible. |
| // If the parameters are known at init-time though, they may be provided. |
| virtual int Initialize() = 0; |
| |
| // The int16 interfaces require: |
| // - only |NativeRate|s be used |
| // - that the input, output and reverse rates must match |
| // - that |processing_config.output_stream()| matches |
| // |processing_config.input_stream()|. |
| // |
| // The float interfaces accept arbitrary rates and support differing input and |
| // output layouts, but the output must have either one channel or the same |
| // number of channels as the input. |
| virtual int Initialize(const ProcessingConfig& processing_config) = 0; |
| |
| // Initialize with unpacked parameters. See Initialize() above for details. |
| // |
| // TODO(mgraczyk): Remove once clients are updated to use the new interface. |
| virtual int Initialize(int capture_input_sample_rate_hz, |
| int capture_output_sample_rate_hz, |
| int render_sample_rate_hz, |
| ChannelLayout capture_input_layout, |
| ChannelLayout capture_output_layout, |
| ChannelLayout render_input_layout) = 0; |
| |
| // TODO(peah): This method is a temporary solution used to take control |
| // over the parameters in the audio processing module and is likely to change. |
| virtual void ApplyConfig(const Config& config) = 0; |
| |
| // Pass down additional options which don't have explicit setters. This |
| // ensures the options are applied immediately. |
| virtual void SetExtraOptions(const webrtc::Config& config) = 0; |
| |
| // TODO(ajm): Only intended for internal use. Make private and friend the |
| // necessary classes? |
| virtual int proc_sample_rate_hz() const = 0; |
| virtual int proc_split_sample_rate_hz() const = 0; |
| virtual size_t num_input_channels() const = 0; |
| virtual size_t num_proc_channels() const = 0; |
| virtual size_t num_output_channels() const = 0; |
| virtual size_t num_reverse_channels() const = 0; |
| |
| // Set to true when the output of AudioProcessing will be muted or in some |
| // other way not used. Ideally, the captured audio would still be processed, |
| // but some components may change behavior based on this information. |
| // Default false. |
| virtual void set_output_will_be_muted(bool muted) = 0; |
| |
| // Processes a 10 ms |frame| of the primary audio stream. On the client-side, |
| // this is the near-end (or captured) audio. |
| // |
| // If needed for enabled functionality, any function with the set_stream_ tag |
| // must be called prior to processing the current frame. Any getter function |
| // with the stream_ tag which is needed should be called after processing. |
| // |
| // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| |
| // members of |frame| must be valid. If changed from the previous call to this |
| // method, it will trigger an initialization. |
| virtual int ProcessStream(AudioFrame* frame) = 0; |
| |
| // Accepts deinterleaved float audio with the range [-1, 1]. Each element |
| // of |src| points to a channel buffer, arranged according to |
| // |input_layout|. At output, the channels will be arranged according to |
| // |output_layout| at |output_sample_rate_hz| in |dest|. |
| // |
| // The output layout must have one channel or as many channels as the input. |
| // |src| and |dest| may use the same memory, if desired. |
| // |
| // TODO(mgraczyk): Remove once clients are updated to use the new interface. |
| virtual int ProcessStream(const float* const* src, |
| size_t samples_per_channel, |
| int input_sample_rate_hz, |
| ChannelLayout input_layout, |
| int output_sample_rate_hz, |
| ChannelLayout output_layout, |
| float* const* dest) = 0; |
| |
| // Accepts deinterleaved float audio with the range [-1, 1]. Each element of |
| // |src| points to a channel buffer, arranged according to |input_stream|. At |
| // output, the channels will be arranged according to |output_stream| in |
| // |dest|. |
| // |
| // The output must have one channel or as many channels as the input. |src| |
| // and |dest| may use the same memory, if desired. |
| virtual int ProcessStream(const float* const* src, |
| const StreamConfig& input_config, |
| const StreamConfig& output_config, |
| float* const* dest) = 0; |
| |
| // Processes a 10 ms |frame| of the reverse direction audio stream. The frame |
| // may be modified. On the client-side, this is the far-end (or to be |
| // rendered) audio. |
| // |
| // It is necessary to provide this if echo processing is enabled, as the |
| // reverse stream forms the echo reference signal. It is recommended, but not |
| // necessary, to provide if gain control is enabled. On the server-side this |
| // typically will not be used. If you're not sure what to pass in here, |
| // chances are you don't need to use it. |
| // |
| // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| |
| // members of |frame| must be valid. |
| virtual int ProcessReverseStream(AudioFrame* frame) = 0; |
| |
| // Accepts deinterleaved float audio with the range [-1, 1]. Each element |
| // of |data| points to a channel buffer, arranged according to |layout|. |
| // TODO(mgraczyk): Remove once clients are updated to use the new interface. |
| virtual int AnalyzeReverseStream(const float* const* data, |
| size_t samples_per_channel, |
| int sample_rate_hz, |
| ChannelLayout layout) = 0; |
| |
| // Accepts deinterleaved float audio with the range [-1, 1]. Each element of |
| // |data| points to a channel buffer, arranged according to |reverse_config|. |
| virtual int ProcessReverseStream(const float* const* src, |
| const StreamConfig& input_config, |
| const StreamConfig& output_config, |
| float* const* dest) = 0; |
| |
| // This must be called if and only if echo processing is enabled. |
| // |
| // Sets the |delay| in ms between ProcessReverseStream() receiving a far-end |
| // frame and ProcessStream() receiving a near-end frame containing the |
| // corresponding echo. On the client-side this can be expressed as |
| // delay = (t_render - t_analyze) + (t_process - t_capture) |
| // where, |
| // - t_analyze is the time a frame is passed to ProcessReverseStream() and |
| // t_render is the time the first sample of the same frame is rendered by |
| // the audio hardware. |
| // - t_capture is the time the first sample of a frame is captured by the |
| // audio hardware and t_process is the time the same frame is passed to |
| // ProcessStream(). |
| virtual int set_stream_delay_ms(int delay) = 0; |
| virtual int stream_delay_ms() const = 0; |
| virtual bool was_stream_delay_set() const = 0; |
| |
| // Call to signal that a key press occurred (true) or did not occur (false) |
| // with this chunk of audio. |
| virtual void set_stream_key_pressed(bool key_pressed) = 0; |
| |
| // Sets a delay |offset| in ms to add to the values passed in through |
| // set_stream_delay_ms(). May be positive or negative. |
| // |
| // Note that this could cause an otherwise valid value passed to |
| // set_stream_delay_ms() to return an error. |
| virtual void set_delay_offset_ms(int offset) = 0; |
| virtual int delay_offset_ms() const = 0; |
| |
| // Attaches provided webrtc::AecDump for recording debugging |
| // information. Log file and maximum file size logic is supposed to |
| // be handled by implementing instance of AecDump. Calling this |
| // method when another AecDump is attached resets the active AecDump |
| // with a new one. This causes the d-tor of the earlier AecDump to |
| // be called. The d-tor call may block until all pending logging |
| // tasks are completed. |
| virtual void AttachAecDump(std::unique_ptr<AecDump> aec_dump) = 0; |
| |
| // If no AecDump is attached, this has no effect. If an AecDump is |
| // attached, it's destructor is called. The d-tor may block until |
| // all pending logging tasks are completed. |
| virtual void DetachAecDump() = 0; |
| |
| // Use to send UMA histograms at end of a call. Note that all histogram |
| // specific member variables are reset. |
| virtual void UpdateHistogramsOnCallEnd() = 0; |
| |
| // TODO(ivoc): Remove when the calling code no longer uses the old Statistics |
| // API. |
| struct Statistic { |
| int instant = 0; // Instantaneous value. |
| int average = 0; // Long-term average. |
| int maximum = 0; // Long-term maximum. |
| int minimum = 0; // Long-term minimum. |
| }; |
| |
| struct Stat { |
| void Set(const Statistic& other) { |
| Set(other.instant, other.average, other.maximum, other.minimum); |
| } |
| void Set(float instant, float average, float maximum, float minimum) { |
| instant_ = instant; |
| average_ = average; |
| maximum_ = maximum; |
| minimum_ = minimum; |
| } |
| float instant() const { return instant_; } |
| float average() const { return average_; } |
| float maximum() const { return maximum_; } |
| float minimum() const { return minimum_; } |
| |
| private: |
| float instant_ = 0.0f; // Instantaneous value. |
| float average_ = 0.0f; // Long-term average. |
| float maximum_ = 0.0f; // Long-term maximum. |
| float minimum_ = 0.0f; // Long-term minimum. |
| }; |
| |
| struct AudioProcessingStatistics { |
| AudioProcessingStatistics(); |
| AudioProcessingStatistics(const AudioProcessingStatistics& other); |
| ~AudioProcessingStatistics(); |
| |
| // AEC Statistics. |
| // RERL = ERL + ERLE |
| Stat residual_echo_return_loss; |
| // ERL = 10log_10(P_far / P_echo) |
| Stat echo_return_loss; |
| // ERLE = 10log_10(P_echo / P_out) |
| Stat echo_return_loss_enhancement; |
| // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a) |
| Stat a_nlp; |
| // Fraction of time that the AEC linear filter is divergent, in a 1-second |
| // non-overlapped aggregation window. |
| float divergent_filter_fraction = -1.0f; |
| |
| // The delay metrics consists of the delay median and standard deviation. It |
| // also consists of the fraction of delay estimates that can make the echo |
| // cancellation perform poorly. The values are aggregated until the first |
| // call to |GetStatistics()| and afterwards aggregated and updated every |
| // second. Note that if there are several clients pulling metrics from |
| // |GetStatistics()| during a session the first call from any of them will |
| // change to one second aggregation window for all. |
| int delay_median = -1; |
| int delay_standard_deviation = -1; |
| float fraction_poor_delays = -1.0f; |
| |
| // Residual echo detector likelihood. |
| float residual_echo_likelihood = -1.0f; |
| // Maximum residual echo likelihood from the last time period. |
| float residual_echo_likelihood_recent_max = -1.0f; |
| }; |
| |
| // TODO(ivoc): Make this pure virtual when all subclasses have been updated. |
| virtual AudioProcessingStatistics GetStatistics() const; |
| |
| // This returns the stats as optionals and it will replace the regular |
| // GetStatistics. |
| virtual AudioProcessingStats GetStatistics(bool has_remote_tracks) const; |
| |
| // These provide access to the component interfaces and should never return |
| // NULL. The pointers will be valid for the lifetime of the APM instance. |
| // The memory for these objects is entirely managed internally. |
| virtual EchoCancellation* echo_cancellation() const = 0; |
| virtual EchoControlMobile* echo_control_mobile() const = 0; |
| virtual GainControl* gain_control() const = 0; |
| // TODO(peah): Deprecate this API call. |
| virtual HighPassFilter* high_pass_filter() const = 0; |
| virtual LevelEstimator* level_estimator() const = 0; |
| virtual NoiseSuppression* noise_suppression() const = 0; |
| virtual VoiceDetection* voice_detection() const = 0; |
| |
| // Returns the last applied configuration. |
| virtual AudioProcessing::Config GetConfig() const = 0; |
| |
| enum Error { |
| // Fatal errors. |
| kNoError = 0, |
| kUnspecifiedError = -1, |
| kCreationFailedError = -2, |
| kUnsupportedComponentError = -3, |
| kUnsupportedFunctionError = -4, |
| kNullPointerError = -5, |
| kBadParameterError = -6, |
| kBadSampleRateError = -7, |
| kBadDataLengthError = -8, |
| kBadNumberChannelsError = -9, |
| kFileError = -10, |
| kStreamParameterNotSetError = -11, |
| kNotEnabledError = -12, |
| |
| // Warnings are non-fatal. |
| // This results when a set_stream_ parameter is out of range. Processing |
| // will continue, but the parameter may have been truncated. |
| kBadStreamParameterWarning = -13 |
| }; |
| |
| enum NativeRate { |
| kSampleRate8kHz = 8000, |
| kSampleRate16kHz = 16000, |
| kSampleRate32kHz = 32000, |
| kSampleRate48kHz = 48000 |
| }; |
| |
| // TODO(kwiberg): We currently need to support a compiler (Visual C++) that |
| // complains if we don't explicitly state the size of the array here. Remove |
| // the size when that's no longer the case. |
| static constexpr int kNativeSampleRatesHz[4] = { |
| kSampleRate8kHz, kSampleRate16kHz, kSampleRate32kHz, kSampleRate48kHz}; |
| static constexpr size_t kNumNativeSampleRates = |
| arraysize(kNativeSampleRatesHz); |
| static constexpr int kMaxNativeSampleRateHz = |
| kNativeSampleRatesHz[kNumNativeSampleRates - 1]; |
| |
| static const int kChunkSizeMs = 10; |
| }; |
| |
| class AudioProcessingBuilder { |
| public: |
| AudioProcessingBuilder(); |
| ~AudioProcessingBuilder(); |
| // The AudioProcessingBuilder takes ownership of the echo_control_factory. |
| AudioProcessingBuilder& SetEchoControlFactory( |
| std::unique_ptr<EchoControlFactory> echo_control_factory); |
| // The AudioProcessingBuilder takes ownership of the capture_post_processing. |
| AudioProcessingBuilder& SetCapturePostProcessing( |
| std::unique_ptr<CustomProcessing> capture_post_processing); |
| // The AudioProcessingBuilder takes ownership of the render_pre_processing. |
| AudioProcessingBuilder& SetRenderPreProcessing( |
| std::unique_ptr<CustomProcessing> render_pre_processing); |
| // The AudioProcessingBuilder takes ownership of the nonlinear beamformer. |
| AudioProcessingBuilder& SetNonlinearBeamformer( |
| std::unique_ptr<NonlinearBeamformer> nonlinear_beamformer); |
| // The AudioProcessingBuilder takes ownership of the echo_detector. |
| AudioProcessingBuilder& SetEchoDetector( |
| std::unique_ptr<EchoDetector> echo_detector); |
| // This creates an APM instance using the previously set components. Calling |
| // the Create function resets the AudioProcessingBuilder to its initial state. |
| AudioProcessing* Create(); |
| AudioProcessing* Create(const webrtc::Config& config); |
| |
| private: |
| std::unique_ptr<EchoControlFactory> echo_control_factory_; |
| std::unique_ptr<CustomProcessing> capture_post_processing_; |
| std::unique_ptr<CustomProcessing> render_pre_processing_; |
| std::unique_ptr<NonlinearBeamformer> nonlinear_beamformer_; |
| std::unique_ptr<EchoDetector> echo_detector_; |
| RTC_DISALLOW_COPY_AND_ASSIGN(AudioProcessingBuilder); |
| }; |
| |
| class StreamConfig { |
| public: |
| // sample_rate_hz: The sampling rate of the stream. |
| // |
| // num_channels: The number of audio channels in the stream, excluding the |
| // keyboard channel if it is present. When passing a |
| // StreamConfig with an array of arrays T*[N], |
| // |
| // N == {num_channels + 1 if has_keyboard |
| // {num_channels if !has_keyboard |
| // |
| // has_keyboard: True if the stream has a keyboard channel. When has_keyboard |
| // is true, the last channel in any corresponding list of |
| // channels is the keyboard channel. |
| StreamConfig(int sample_rate_hz = 0, |
| size_t num_channels = 0, |
| bool has_keyboard = false) |
| : sample_rate_hz_(sample_rate_hz), |
| num_channels_(num_channels), |
| has_keyboard_(has_keyboard), |
| num_frames_(calculate_frames(sample_rate_hz)) {} |
| |
| void set_sample_rate_hz(int value) { |
| sample_rate_hz_ = value; |
| num_frames_ = calculate_frames(value); |
| } |
| void set_num_channels(size_t value) { num_channels_ = value; } |
| void set_has_keyboard(bool value) { has_keyboard_ = value; } |
| |
| int sample_rate_hz() const { return sample_rate_hz_; } |
| |
| // The number of channels in the stream, not including the keyboard channel if |
| // present. |
| size_t num_channels() const { return num_channels_; } |
| |
| bool has_keyboard() const { return has_keyboard_; } |
| size_t num_frames() const { return num_frames_; } |
| size_t num_samples() const { return num_channels_ * num_frames_; } |
| |
| bool operator==(const StreamConfig& other) const { |
| return sample_rate_hz_ == other.sample_rate_hz_ && |
| num_channels_ == other.num_channels_ && |
| has_keyboard_ == other.has_keyboard_; |
| } |
| |
| bool operator!=(const StreamConfig& other) const { return !(*this == other); } |
| |
| private: |
| static size_t calculate_frames(int sample_rate_hz) { |
| return static_cast<size_t>( |
| AudioProcessing::kChunkSizeMs * sample_rate_hz / 1000); |
| } |
| |
| int sample_rate_hz_; |
| size_t num_channels_; |
| bool has_keyboard_; |
| size_t num_frames_; |
| }; |
| |
| class ProcessingConfig { |
| public: |
| enum StreamName { |
| kInputStream, |
| kOutputStream, |
| kReverseInputStream, |
| kReverseOutputStream, |
| kNumStreamNames, |
| }; |
| |
| const StreamConfig& input_stream() const { |
| return streams[StreamName::kInputStream]; |
| } |
| const StreamConfig& output_stream() const { |
| return streams[StreamName::kOutputStream]; |
| } |
| const StreamConfig& reverse_input_stream() const { |
| return streams[StreamName::kReverseInputStream]; |
| } |
| const StreamConfig& reverse_output_stream() const { |
| return streams[StreamName::kReverseOutputStream]; |
| } |
| |
| StreamConfig& input_stream() { return streams[StreamName::kInputStream]; } |
| StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; } |
| StreamConfig& reverse_input_stream() { |
| return streams[StreamName::kReverseInputStream]; |
| } |
| StreamConfig& reverse_output_stream() { |
| return streams[StreamName::kReverseOutputStream]; |
| } |
| |
| bool operator==(const ProcessingConfig& other) const { |
| for (int i = 0; i < StreamName::kNumStreamNames; ++i) { |
| if (this->streams[i] != other.streams[i]) { |
| return false; |
| } |
| } |
| return true; |
| } |
| |
| bool operator!=(const ProcessingConfig& other) const { |
| return !(*this == other); |
| } |
| |
| StreamConfig streams[StreamName::kNumStreamNames]; |
| }; |
| |
| // The acoustic echo cancellation (AEC) component provides better performance |
| // than AECM but also requires more processing power and is dependent on delay |
| // stability and reporting accuracy. As such it is well-suited and recommended |
| // for PC and IP phone applications. |
| // |
| // Not recommended to be enabled on the server-side. |
| class EchoCancellation { |
| public: |
| // EchoCancellation and EchoControlMobile may not be enabled simultaneously. |
| // Enabling one will disable the other. |
| virtual int Enable(bool enable) = 0; |
| virtual bool is_enabled() const = 0; |
| |
| // Differences in clock speed on the primary and reverse streams can impact |
| // the AEC performance. On the client-side, this could be seen when different |
| // render and capture devices are used, particularly with webcams. |
| // |
| // This enables a compensation mechanism, and requires that |
| // set_stream_drift_samples() be called. |
| virtual int enable_drift_compensation(bool enable) = 0; |
| virtual bool is_drift_compensation_enabled() const = 0; |
| |
| // Sets the difference between the number of samples rendered and captured by |
| // the audio devices since the last call to |ProcessStream()|. Must be called |
| // if drift compensation is enabled, prior to |ProcessStream()|. |
| virtual void set_stream_drift_samples(int drift) = 0; |
| virtual int stream_drift_samples() const = 0; |
| |
| enum SuppressionLevel { |
| kLowSuppression, |
| kModerateSuppression, |
| kHighSuppression |
| }; |
| |
| // Sets the aggressiveness of the suppressor. A higher level trades off |
| // double-talk performance for increased echo suppression. |
| virtual int set_suppression_level(SuppressionLevel level) = 0; |
| virtual SuppressionLevel suppression_level() const = 0; |
| |
| // Returns false if the current frame almost certainly contains no echo |
| // and true if it _might_ contain echo. |
| virtual bool stream_has_echo() const = 0; |
| |
| // Enables the computation of various echo metrics. These are obtained |
| // through |GetMetrics()|. |
| virtual int enable_metrics(bool enable) = 0; |
| virtual bool are_metrics_enabled() const = 0; |
| |
| // Each statistic is reported in dB. |
| // P_far: Far-end (render) signal power. |
| // P_echo: Near-end (capture) echo signal power. |
| // P_out: Signal power at the output of the AEC. |
| // P_a: Internal signal power at the point before the AEC's non-linear |
| // processor. |
| struct Metrics { |
| // RERL = ERL + ERLE |
| AudioProcessing::Statistic residual_echo_return_loss; |
| |
| // ERL = 10log_10(P_far / P_echo) |
| AudioProcessing::Statistic echo_return_loss; |
| |
| // ERLE = 10log_10(P_echo / P_out) |
| AudioProcessing::Statistic echo_return_loss_enhancement; |
| |
| // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a) |
| AudioProcessing::Statistic a_nlp; |
| |
| // Fraction of time that the AEC linear filter is divergent, in a 1-second |
| // non-overlapped aggregation window. |
| float divergent_filter_fraction; |
| }; |
| |
| // Deprecated. Use GetStatistics on the AudioProcessing interface instead. |
| // TODO(ajm): discuss the metrics update period. |
| virtual int GetMetrics(Metrics* metrics) = 0; |
| |
| // Enables computation and logging of delay values. Statistics are obtained |
| // through |GetDelayMetrics()|. |
| virtual int enable_delay_logging(bool enable) = 0; |
| virtual bool is_delay_logging_enabled() const = 0; |
| |
| // The delay metrics consists of the delay |median| and the delay standard |
| // deviation |std|. It also consists of the fraction of delay estimates |
| // |fraction_poor_delays| that can make the echo cancellation perform poorly. |
| // The values are aggregated until the first call to |GetDelayMetrics()| and |
| // afterwards aggregated and updated every second. |
| // Note that if there are several clients pulling metrics from |
| // |GetDelayMetrics()| during a session the first call from any of them will |
| // change to one second aggregation window for all. |
| // Deprecated. Use GetStatistics on the AudioProcessing interface instead. |
| virtual int GetDelayMetrics(int* median, int* std) = 0; |
| // Deprecated. Use GetStatistics on the AudioProcessing interface instead. |
| virtual int GetDelayMetrics(int* median, int* std, |
| float* fraction_poor_delays) = 0; |
| |
| // Returns a pointer to the low level AEC component. In case of multiple |
| // channels, the pointer to the first one is returned. A NULL pointer is |
| // returned when the AEC component is disabled or has not been initialized |
| // successfully. |
| virtual struct AecCore* aec_core() const = 0; |
| |
| protected: |
| virtual ~EchoCancellation() {} |
| }; |
| |
| // The acoustic echo control for mobile (AECM) component is a low complexity |
| // robust option intended for use on mobile devices. |
| // |
| // Not recommended to be enabled on the server-side. |
| class EchoControlMobile { |
| public: |
| // EchoCancellation and EchoControlMobile may not be enabled simultaneously. |
| // Enabling one will disable the other. |
| virtual int Enable(bool enable) = 0; |
| virtual bool is_enabled() const = 0; |
| |
| // Recommended settings for particular audio routes. In general, the louder |
| // the echo is expected to be, the higher this value should be set. The |
| // preferred setting may vary from device to device. |
| enum RoutingMode { |
| kQuietEarpieceOrHeadset, |
| kEarpiece, |
| kLoudEarpiece, |
| kSpeakerphone, |
| kLoudSpeakerphone |
| }; |
| |
| // Sets echo control appropriate for the audio routing |mode| on the device. |
| // It can and should be updated during a call if the audio routing changes. |
| virtual int set_routing_mode(RoutingMode mode) = 0; |
| virtual RoutingMode routing_mode() const = 0; |
| |
| // Comfort noise replaces suppressed background noise to maintain a |
| // consistent signal level. |
| virtual int enable_comfort_noise(bool enable) = 0; |
| virtual bool is_comfort_noise_enabled() const = 0; |
| |
| // A typical use case is to initialize the component with an echo path from a |
| // previous call. The echo path is retrieved using |GetEchoPath()|, typically |
| // at the end of a call. The data can then be stored for later use as an |
| // initializer before the next call, using |SetEchoPath()|. |
| // |
| // Controlling the echo path this way requires the data |size_bytes| to match |
| // the internal echo path size. This size can be acquired using |
| // |echo_path_size_bytes()|. |SetEchoPath()| causes an entire reset, worth |
| // noting if it is to be called during an ongoing call. |
| // |
| // It is possible that version incompatibilities may result in a stored echo |
| // path of the incorrect size. In this case, the stored path should be |
| // discarded. |
| virtual int SetEchoPath(const void* echo_path, size_t size_bytes) = 0; |
| virtual int GetEchoPath(void* echo_path, size_t size_bytes) const = 0; |
| |
| // The returned path size is guaranteed not to change for the lifetime of |
| // the application. |
| static size_t echo_path_size_bytes(); |
| |
| protected: |
| virtual ~EchoControlMobile() {} |
| }; |
| |
| // The automatic gain control (AGC) component brings the signal to an |
| // appropriate range. This is done by applying a digital gain directly and, in |
| // the analog mode, prescribing an analog gain to be applied at the audio HAL. |
| // |
| // Recommended to be enabled on the client-side. |
| class GainControl { |
| public: |
| virtual int Enable(bool enable) = 0; |
| virtual bool is_enabled() const = 0; |
| |
| // When an analog mode is set, this must be called prior to |ProcessStream()| |
| // to pass the current analog level from the audio HAL. Must be within the |
| // range provided to |set_analog_level_limits()|. |
| virtual int set_stream_analog_level(int level) = 0; |
| |
| // When an analog mode is set, this should be called after |ProcessStream()| |
| // to obtain the recommended new analog level for the audio HAL. It is the |
| // users responsibility to apply this level. |
| virtual int stream_analog_level() = 0; |
| |
| enum Mode { |
| // Adaptive mode intended for use if an analog volume control is available |
| // on the capture device. It will require the user to provide coupling |
| // between the OS mixer controls and AGC through the |stream_analog_level()| |
| // functions. |
| // |
| // It consists of an analog gain prescription for the audio device and a |
| // digital compression stage. |
| kAdaptiveAnalog, |
| |
| // Adaptive mode intended for situations in which an analog volume control |
| // is unavailable. It operates in a similar fashion to the adaptive analog |
| // mode, but with scaling instead applied in the digital domain. As with |
| // the analog mode, it additionally uses a digital compression stage. |
| kAdaptiveDigital, |
| |
| // Fixed mode which enables only the digital compression stage also used by |
| // the two adaptive modes. |
| // |
| // It is distinguished from the adaptive modes by considering only a |
| // short time-window of the input signal. It applies a fixed gain through |
| // most of the input level range, and compresses (gradually reduces gain |
| // with increasing level) the input signal at higher levels. This mode is |
| // preferred on embedded devices where the capture signal level is |
| // predictable, so that a known gain can be applied. |
| kFixedDigital |
| }; |
| |
| virtual int set_mode(Mode mode) = 0; |
| virtual Mode mode() const = 0; |
| |
| // Sets the target peak |level| (or envelope) of the AGC in dBFs (decibels |
| // from digital full-scale). The convention is to use positive values. For |
| // instance, passing in a value of 3 corresponds to -3 dBFs, or a target |
| // level 3 dB below full-scale. Limited to [0, 31]. |
| // |
| // TODO(ajm): use a negative value here instead, if/when VoE will similarly |
| // update its interface. |
| virtual int set_target_level_dbfs(int level) = 0; |
| virtual int target_level_dbfs() const = 0; |
| |
| // Sets the maximum |gain| the digital compression stage may apply, in dB. A |
| // higher number corresponds to greater compression, while a value of 0 will |
| // leave the signal uncompressed. Limited to [0, 90]. |
| virtual int set_compression_gain_db(int gain) = 0; |
| virtual int compression_gain_db() const = 0; |
| |
| // When enabled, the compression stage will hard limit the signal to the |
| // target level. Otherwise, the signal will be compressed but not limited |
| // above the target level. |
| virtual int enable_limiter(bool enable) = 0; |
| virtual bool is_limiter_enabled() const = 0; |
| |
| // Sets the |minimum| and |maximum| analog levels of the audio capture device. |
| // Must be set if and only if an analog mode is used. Limited to [0, 65535]. |
| virtual int set_analog_level_limits(int minimum, |
| int maximum) = 0; |
| virtual int analog_level_minimum() const = 0; |
| virtual int analog_level_maximum() const = 0; |
| |
| // Returns true if the AGC has detected a saturation event (period where the |
| // signal reaches digital full-scale) in the current frame and the analog |
| // level cannot be reduced. |
| // |
| // This could be used as an indicator to reduce or disable analog mic gain at |
| // the audio HAL. |
| virtual bool stream_is_saturated() const = 0; |
| |
| protected: |
| virtual ~GainControl() {} |
| }; |
| // TODO(peah): Remove this interface. |
| // A filtering component which removes DC offset and low-frequency noise. |
| // Recommended to be enabled on the client-side. |
| class HighPassFilter { |
| public: |
| virtual int Enable(bool enable) = 0; |
| virtual bool is_enabled() const = 0; |
| |
| virtual ~HighPassFilter() {} |
| }; |
| |
| // An estimation component used to retrieve level metrics. |
| class LevelEstimator { |
| public: |
| virtual int Enable(bool enable) = 0; |
| virtual bool is_enabled() const = 0; |
| |
| // Returns the root mean square (RMS) level in dBFs (decibels from digital |
| // full-scale), or alternately dBov. It is computed over all primary stream |
| // frames since the last call to RMS(). The returned value is positive but |
| // should be interpreted as negative. It is constrained to [0, 127]. |
| // |
| // The computation follows: https://tools.ietf.org/html/rfc6465 |
| // with the intent that it can provide the RTP audio level indication. |
| // |
| // Frames passed to ProcessStream() with an |_energy| of zero are considered |
| // to have been muted. The RMS of the frame will be interpreted as -127. |
| virtual int RMS() = 0; |
| |
| protected: |
| virtual ~LevelEstimator() {} |
| }; |
| |
| // The noise suppression (NS) component attempts to remove noise while |
| // retaining speech. Recommended to be enabled on the client-side. |
| // |
| // Recommended to be enabled on the client-side. |
| class NoiseSuppression { |
| public: |
| virtual int Enable(bool enable) = 0; |
| virtual bool is_enabled() const = 0; |
| |
| // Determines the aggressiveness of the suppression. Increasing the level |
| // will reduce the noise level at the expense of a higher speech distortion. |
| enum Level { |
| kLow, |
| kModerate, |
| kHigh, |
| kVeryHigh |
| }; |
| |
| virtual int set_level(Level level) = 0; |
| virtual Level level() const = 0; |
| |
| // Returns the internally computed prior speech probability of current frame |
| // averaged over output channels. This is not supported in fixed point, for |
| // which |kUnsupportedFunctionError| is returned. |
| virtual float speech_probability() const = 0; |
| |
| // Returns the noise estimate per frequency bin averaged over all channels. |
| virtual std::vector<float> NoiseEstimate() = 0; |
| |
| protected: |
| virtual ~NoiseSuppression() {} |
| }; |
| |
| // Interface for a custom processing submodule. |
| class CustomProcessing { |
| public: |
| // (Re-)Initializes the submodule. |
| virtual void Initialize(int sample_rate_hz, int num_channels) = 0; |
| // Processes the given capture or render signal. |
| virtual void Process(AudioBuffer* audio) = 0; |
| // Returns a string representation of the module state. |
| virtual std::string ToString() const = 0; |
| |
| virtual ~CustomProcessing() {} |
| }; |
| |
| // Interface for an echo detector submodule. |
| class EchoDetector { |
| public: |
| // (Re-)Initializes the submodule. |
| virtual void Initialize(int sample_rate_hz, int num_channels) = 0; |
| |
| // Analysis (not changing) of the render signal. |
| virtual void AnalyzeRenderAudio(rtc::ArrayView<const float> render_audio) = 0; |
| |
| // Analysis (not changing) of the capture signal. |
| virtual void AnalyzeCaptureAudio( |
| rtc::ArrayView<const float> capture_audio) = 0; |
| |
| // Pack an AudioBuffer into a vector<float>. |
| static void PackRenderAudioBuffer(AudioBuffer* audio, |
| std::vector<float>* packed_buffer); |
| |
| struct Metrics { |
| double echo_likelihood; |
| double echo_likelihood_recent_max; |
| }; |
| |
| // Collect current metrics from the echo detector. |
| virtual Metrics GetMetrics() const = 0; |
| |
| virtual ~EchoDetector() {} |
| }; |
| |
| // The voice activity detection (VAD) component analyzes the stream to |
| // determine if voice is present. A facility is also provided to pass in an |
| // external VAD decision. |
| // |
| // In addition to |stream_has_voice()| the VAD decision is provided through the |
| // |AudioFrame| passed to |ProcessStream()|. The |vad_activity_| member will be |
| // modified to reflect the current decision. |
| class VoiceDetection { |
| public: |
| virtual int Enable(bool enable) = 0; |
| virtual bool is_enabled() const = 0; |
| |
| // Returns true if voice is detected in the current frame. Should be called |
| // after |ProcessStream()|. |
| virtual bool stream_has_voice() const = 0; |
| |
| // Some of the APM functionality requires a VAD decision. In the case that |
| // a decision is externally available for the current frame, it can be passed |
| // in here, before |ProcessStream()| is called. |
| // |
| // VoiceDetection does _not_ need to be enabled to use this. If it happens to |
| // be enabled, detection will be skipped for any frame in which an external |
| // VAD decision is provided. |
| virtual int set_stream_has_voice(bool has_voice) = 0; |
| |
| // Specifies the likelihood that a frame will be declared to contain voice. |
| // A higher value makes it more likely that speech will not be clipped, at |
| // the expense of more noise being detected as voice. |
| enum Likelihood { |
| kVeryLowLikelihood, |
| kLowLikelihood, |
| kModerateLikelihood, |
| kHighLikelihood |
| }; |
| |
| virtual int set_likelihood(Likelihood likelihood) = 0; |
| virtual Likelihood likelihood() const = 0; |
| |
| // Sets the |size| of the frames in ms on which the VAD will operate. Larger |
| // frames will improve detection accuracy, but reduce the frequency of |
| // updates. |
| // |
| // This does not impact the size of frames passed to |ProcessStream()|. |
| virtual int set_frame_size_ms(int size) = 0; |
| virtual int frame_size_ms() const = 0; |
| |
| protected: |
| virtual ~VoiceDetection() {} |
| }; |
| |
| class EchoCanceller3Factory : public EchoControlFactory { |
| public: |
| EchoCanceller3Factory(); |
| EchoCanceller3Factory(const EchoCanceller3Config& config); |
| std::unique_ptr<EchoControl> Create(int sample_rate_hz) override; |
| |
| private: |
| EchoCanceller3Config config_; |
| }; |
| } // namespace webrtc |
| |
| #endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |