Allow an external audio processing module to be used in WebRTC
[This CL is a rebase of an original CL by solenberg@:
https://codereview.webrtc.org/2948763002/ which in turn was a
rebase of an original CL by peah@:
https://chromium-review.googlesource.com/c/527032/]
Allow an external audio processing module to be used in WebRTC
This CL adds support for optionally using an externally created audio
processing module in a peerconnection. The ownership is shared
between the peerconnection and the external creator of the module.
As part of this the internal ownership of the audio processing module
is moved from VoiceEngine to WebRtcVoiceEngine.
BUG=webrtc:7775
Review-Url: https://codereview.webrtc.org/2961723004
Cr-Commit-Position: refs/heads/master@{#18837}
diff --git a/webrtc/audio/audio_receive_stream_unittest.cc b/webrtc/audio/audio_receive_stream_unittest.cc
index 84efb20..127ea07 100644
--- a/webrtc/audio/audio_receive_stream_unittest.cc
+++ b/webrtc/audio/audio_receive_stream_unittest.cc
@@ -17,6 +17,7 @@
#include "webrtc/audio/conversion.h"
#include "webrtc/call/rtp_stream_receiver_controller.h"
#include "webrtc/logging/rtc_event_log/mock/mock_rtc_event_log.h"
+#include "webrtc/modules/audio_processing/include/mock_audio_processing.h"
#include "webrtc/modules/bitrate_controller/include/mock/mock_bitrate_controller.h"
#include "webrtc/modules/pacing/packet_router.h"
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
@@ -74,13 +75,13 @@
RegisterVoiceEngineObserver(_)).WillOnce(Return(0));
EXPECT_CALL(voice_engine_,
DeRegisterVoiceEngineObserver()).WillOnce(Return(0));
- EXPECT_CALL(voice_engine_, audio_processing());
EXPECT_CALL(voice_engine_, audio_device_module());
EXPECT_CALL(voice_engine_, audio_transport());
AudioState::Config config;
config.voice_engine = &voice_engine_;
config.audio_mixer = audio_mixer_;
+ config.audio_processing = new rtc::RefCountedObject<MockAudioProcessing>();
audio_state_ = AudioState::Create(config);
EXPECT_CALL(voice_engine_, ChannelProxyFactory(kChannelId))
diff --git a/webrtc/audio/audio_send_stream.cc b/webrtc/audio/audio_send_stream.cc
index f89da98..f8ee3ab 100644
--- a/webrtc/audio/audio_send_stream.cc
+++ b/webrtc/audio/audio_send_stream.cc
@@ -279,8 +279,9 @@
stats.audio_level = base->transmit_mixer()->AudioLevelFullRange();
RTC_DCHECK_LE(0, stats.audio_level);
- RTC_DCHECK(base->audio_processing());
- auto audio_processing_stats = base->audio_processing()->GetStatistics();
+ RTC_DCHECK(audio_state_->audio_processing());
+ auto audio_processing_stats =
+ audio_state_->audio_processing()->GetStatistics();
stats.echo_delay_median_ms = audio_processing_stats.delay_median;
stats.echo_delay_std_ms = audio_processing_stats.delay_standard_deviation;
stats.echo_return_loss = audio_processing_stats.echo_return_loss.instant();
diff --git a/webrtc/audio/audio_send_stream_unittest.cc b/webrtc/audio/audio_send_stream_unittest.cc
index e6acf92..efc18d1 100644
--- a/webrtc/audio/audio_send_stream_unittest.cc
+++ b/webrtc/audio/audio_send_stream_unittest.cc
@@ -128,6 +128,7 @@
struct ConfigHelper {
ConfigHelper(bool audio_bwe_enabled, bool expect_set_encoder_call)
: stream_config_(nullptr),
+ audio_processing_(new rtc::RefCountedObject<MockAudioProcessing>()),
simulated_clock_(123456),
send_side_cc_(rtc::MakeUnique<SendSideCongestionController>(
&simulated_clock_,
@@ -144,12 +145,12 @@
EXPECT_CALL(voice_engine_,
DeRegisterVoiceEngineObserver()).WillOnce(Return(0));
EXPECT_CALL(voice_engine_, audio_device_module());
- EXPECT_CALL(voice_engine_, audio_processing());
EXPECT_CALL(voice_engine_, audio_transport());
AudioState::Config config;
config.voice_engine = &voice_engine_;
config.audio_mixer = AudioMixerImpl::Create();
+ config.audio_processing = audio_processing_;
audio_state_ = AudioState::Create(config);
SetupDefaultChannelProxy(audio_bwe_enabled);
@@ -278,8 +279,6 @@
.WillRepeatedly(Return(report_blocks));
EXPECT_CALL(voice_engine_, transmit_mixer())
.WillRepeatedly(Return(&transmit_mixer_));
- EXPECT_CALL(voice_engine_, audio_processing())
- .WillRepeatedly(Return(&audio_processing_));
EXPECT_CALL(transmit_mixer_, AudioLevelFullRange())
.WillRepeatedly(Return(kSpeechInputLevel));
@@ -294,7 +293,7 @@
audio_processing_stats_.delay_median = kEchoDelayMedian;
audio_processing_stats_.delay_standard_deviation = kEchoDelayStdDev;
- EXPECT_CALL(audio_processing_, GetStatistics())
+ EXPECT_CALL(*audio_processing_, GetStatistics())
.WillRepeatedly(Return(audio_processing_stats_));
}
@@ -303,7 +302,7 @@
rtc::scoped_refptr<AudioState> audio_state_;
AudioSendStream::Config stream_config_;
testing::StrictMock<MockVoEChannelProxy>* channel_proxy_ = nullptr;
- MockAudioProcessing audio_processing_;
+ rtc::scoped_refptr<MockAudioProcessing> audio_processing_;
MockTransmitMixer transmit_mixer_;
AudioProcessing::AudioProcessingStatistics audio_processing_stats_;
SimulatedClock simulated_clock_;
diff --git a/webrtc/audio/audio_state.cc b/webrtc/audio/audio_state.cc
index c15ddd7..961e772 100644
--- a/webrtc/audio/audio_state.cc
+++ b/webrtc/audio/audio_state.cc
@@ -19,11 +19,15 @@
namespace webrtc {
namespace internal {
+// TODO(peah): Remove the conditional in the audio_transport_proxy_ constructor
+// call when upstream dependencies have properly been resolved.
AudioState::AudioState(const AudioState::Config& config)
: config_(config),
voe_base_(config.voice_engine),
audio_transport_proxy_(voe_base_->audio_transport(),
- voe_base_->audio_processing(),
+ config_.audio_processing
+ ? config_.audio_processing.get()
+ : voe_base_->audio_processing(),
config_.audio_mixer) {
process_thread_checker_.DetachFromThread();
RTC_DCHECK(config_.audio_mixer);
diff --git a/webrtc/audio/audio_state.h b/webrtc/audio/audio_state.h
index 29b5a9e..8a98d57 100644
--- a/webrtc/audio/audio_state.h
+++ b/webrtc/audio/audio_state.h
@@ -28,8 +28,14 @@
explicit AudioState(const AudioState::Config& config);
~AudioState() override;
- VoiceEngine* voice_engine();
+ // TODO(peah): Remove the conditional when upstream dependencies have properly
+ // been resolved.
+ AudioProcessing* audio_processing() override {
+ return config_.audio_processing ? config_.audio_processing.get()
+ : voe_base_->audio_processing();
+ }
+ VoiceEngine* voice_engine();
rtc::scoped_refptr<AudioMixer> mixer();
bool typing_noise_detected() const;
diff --git a/webrtc/audio/audio_state_unittest.cc b/webrtc/audio/audio_state_unittest.cc
index 05c5003..ffaacb6 100644
--- a/webrtc/audio/audio_state_unittest.cc
+++ b/webrtc/audio/audio_state_unittest.cc
@@ -12,6 +12,7 @@
#include "webrtc/audio/audio_state.h"
#include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
+#include "webrtc/modules/audio_processing/include/mock_audio_processing.h"
#include "webrtc/test/gtest.h"
#include "webrtc/test/mock_voice_engine.h"
@@ -31,8 +32,6 @@
.WillOnce(testing::Return(0));
EXPECT_CALL(mock_voice_engine, audio_device_module())
.Times(testing::AtLeast(1));
- EXPECT_CALL(mock_voice_engine, audio_processing())
- .Times(testing::AtLeast(1));
EXPECT_CALL(mock_voice_engine, audio_transport())
.WillRepeatedly(testing::Return(&audio_transport));
@@ -49,6 +48,8 @@
audio_state_config.voice_engine = &mock_voice_engine;
audio_state_config.audio_mixer = audio_mixer;
+ audio_state_config.audio_processing =
+ new rtc::RefCountedObject<MockAudioProcessing>();
}
AudioState::Config& config() { return audio_state_config; }
MockVoiceEngine& voice_engine() { return mock_voice_engine; }
diff --git a/webrtc/audio/audio_transport_proxy.cc b/webrtc/audio/audio_transport_proxy.cc
index d6ce939..73b0091 100644
--- a/webrtc/audio/audio_transport_proxy.cc
+++ b/webrtc/audio/audio_transport_proxy.cc
@@ -34,11 +34,13 @@
} // namespace
AudioTransportProxy::AudioTransportProxy(AudioTransport* voe_audio_transport,
- AudioProcessing* apm,
+ AudioProcessing* audio_processing,
AudioMixer* mixer)
- : voe_audio_transport_(voe_audio_transport), apm_(apm), mixer_(mixer) {
+ : voe_audio_transport_(voe_audio_transport),
+ audio_processing_(audio_processing),
+ mixer_(mixer) {
RTC_DCHECK(voe_audio_transport);
- RTC_DCHECK(apm);
+ RTC_DCHECK(audio_processing);
RTC_DCHECK(mixer);
}
@@ -85,7 +87,7 @@
*elapsed_time_ms = mixed_frame_.elapsed_time_ms_;
*ntp_time_ms = mixed_frame_.ntp_time_ms_;
- const auto error = apm_->ProcessReverseStream(&mixed_frame_);
+ const auto error = audio_processing_->ProcessReverseStream(&mixed_frame_);
RTC_DCHECK_EQ(error, AudioProcessing::kNoError);
nSamplesOut = Resample(mixed_frame_, samplesPerSec, &resampler_,
diff --git a/webrtc/audio/audio_transport_proxy.h b/webrtc/audio/audio_transport_proxy.h
index 1d03378..fda9339 100644
--- a/webrtc/audio/audio_transport_proxy.h
+++ b/webrtc/audio/audio_transport_proxy.h
@@ -23,7 +23,7 @@
class AudioTransportProxy : public AudioTransport {
public:
AudioTransportProxy(AudioTransport* voe_audio_transport,
- AudioProcessing* apm,
+ AudioProcessing* audio_processing,
AudioMixer* mixer);
~AudioTransportProxy() override;
@@ -65,7 +65,7 @@
private:
AudioTransport* voe_audio_transport_;
- AudioProcessing* apm_;
+ AudioProcessing* audio_processing_;
rtc::scoped_refptr<AudioMixer> mixer_;
AudioFrame mixed_frame_;
// Converts mixed audio to the audio device output rate.
