| /* |
| * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "voice_engine/transmit_mixer.h" |
| |
| #include <memory> |
| |
| #include "audio/utility/audio_frame_operations.h" |
| #include "rtc_base/format_macros.h" |
| #include "rtc_base/location.h" |
| #include "rtc_base/logging.h" |
| #include "system_wrappers/include/event_wrapper.h" |
| #include "system_wrappers/include/trace.h" |
| #include "voice_engine/channel.h" |
| #include "voice_engine/channel_manager.h" |
| #include "voice_engine/statistics.h" |
| #include "voice_engine/utility.h" |
| #include "voice_engine/voe_base_impl.h" |
| |
| namespace webrtc { |
| namespace voe { |
| |
| #if WEBRTC_VOICE_ENGINE_TYPING_DETECTION |
| // TODO(ajm): The thread safety of this is dubious... |
| void TransmitMixer::OnPeriodicProcess() |
| { |
| WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::OnPeriodicProcess()"); |
| |
| bool send_typing_noise_warning = false; |
| bool typing_noise_detected = false; |
| { |
| rtc::CritScope cs(&_critSect); |
| if (_typingNoiseWarningPending) { |
| send_typing_noise_warning = true; |
| typing_noise_detected = _typingNoiseDetected; |
| _typingNoiseWarningPending = false; |
| } |
| } |
| if (send_typing_noise_warning) { |
| rtc::CritScope cs(&_callbackCritSect); |
| if (_voiceEngineObserverPtr) { |
| if (typing_noise_detected) { |
| WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::OnPeriodicProcess() => " |
| "CallbackOnError(VE_TYPING_NOISE_WARNING)"); |
| _voiceEngineObserverPtr->CallbackOnError( |
| -1, |
| VE_TYPING_NOISE_WARNING); |
| } else { |
| WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::OnPeriodicProcess() => " |
| "CallbackOnError(VE_TYPING_NOISE_OFF_WARNING)"); |
| _voiceEngineObserverPtr->CallbackOnError( |
| -1, |
| VE_TYPING_NOISE_OFF_WARNING); |
| } |
| } |
| } |
| } |
| #endif // WEBRTC_VOICE_ENGINE_TYPING_DETECTION |
| |
| int32_t |
| TransmitMixer::Create(TransmitMixer*& mixer, uint32_t instanceId) |
| { |
| WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(instanceId, -1), |
| "TransmitMixer::Create(instanceId=%d)", instanceId); |
| mixer = new TransmitMixer(instanceId); |
| if (mixer == NULL) |
| { |
| WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(instanceId, -1), |
| "TransmitMixer::Create() unable to allocate memory" |
| "for mixer"); |
| return -1; |
| } |
| return 0; |
| } |
| |
| void |
| TransmitMixer::Destroy(TransmitMixer*& mixer) |
| { |
| if (mixer) |
| { |
| delete mixer; |
| mixer = NULL; |
| } |
| } |
| |
| TransmitMixer::TransmitMixer(uint32_t instanceId) : |
| #if WEBRTC_VOICE_ENGINE_TYPING_DETECTION |
| _monitorModule(this), |
| #endif |
| _instanceId(instanceId) |
| { |
| WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::TransmitMixer() - ctor"); |
| } |
| |
| TransmitMixer::~TransmitMixer() |
| { |
| WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::~TransmitMixer() - dtor"); |
| #if WEBRTC_VOICE_ENGINE_TYPING_DETECTION |
| if (_processThreadPtr) |
| _processThreadPtr->DeRegisterModule(&_monitorModule); |
| #endif |
| } |
| |
| int32_t |
| TransmitMixer::SetEngineInformation(ProcessThread& processThread, |
| Statistics& engineStatistics, |
| ChannelManager& channelManager) |
| { |
| WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::SetEngineInformation()"); |
| |
| _processThreadPtr = &processThread; |
| _engineStatisticsPtr = &engineStatistics; |
| _channelManagerPtr = &channelManager; |
| |
| #if WEBRTC_VOICE_ENGINE_TYPING_DETECTION |
| _processThreadPtr->RegisterModule(&_monitorModule, RTC_FROM_HERE); |
| #endif |
| return 0; |
| } |
| |
| int32_t |
| TransmitMixer::RegisterVoiceEngineObserver(VoiceEngineObserver& observer) |
| { |
| WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::RegisterVoiceEngineObserver()"); |
| rtc::CritScope cs(&_callbackCritSect); |
| |
| if (_voiceEngineObserverPtr) |
| { |
| _engineStatisticsPtr->SetLastError( |
| VE_INVALID_OPERATION, kTraceError, |
| "RegisterVoiceEngineObserver() observer already enabled"); |
| return -1; |
| } |
| _voiceEngineObserverPtr = &observer; |
| return 0; |
| } |
| |
| int32_t |
| TransmitMixer::SetAudioProcessingModule(AudioProcessing* audioProcessingModule) |
| { |
| WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::SetAudioProcessingModule(" |
| "audioProcessingModule=0x%x)", |
| audioProcessingModule); |
| audioproc_ = audioProcessingModule; |
| return 0; |
| } |
| |
| void TransmitMixer::GetSendCodecInfo(int* max_sample_rate, |
| size_t* max_channels) { |
| *max_sample_rate = 8000; |
| *max_channels = 1; |
| for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); |
| it.Increment()) { |
| Channel* channel = it.GetChannel(); |
| if (channel->Sending()) { |
| CodecInst codec; |
| // TODO(ossu): Investigate how this could happen. b/62909493 |
| if (channel->GetSendCodec(codec) == 0) { |
| *max_sample_rate = std::max(*max_sample_rate, codec.plfreq); |
| *max_channels = std::max(*max_channels, codec.channels); |
| } else { |
| LOG(LS_WARNING) << "Unable to get send codec for channel " |
| << channel->ChannelId(); |
| RTC_NOTREACHED(); |
| } |
| } |
| } |
| } |
| |
| int32_t |
| TransmitMixer::PrepareDemux(const void* audioSamples, |
| size_t nSamples, |
| size_t nChannels, |
| uint32_t samplesPerSec, |
| uint16_t totalDelayMS, |
| int32_t clockDrift, |
| uint16_t currentMicLevel, |
| bool keyPressed) |
| { |
| WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::PrepareDemux(nSamples=%" PRIuS ", " |
| "nChannels=%" PRIuS ", samplesPerSec=%u, totalDelayMS=%u, " |
| "clockDrift=%d, currentMicLevel=%u)", |
| nSamples, nChannels, samplesPerSec, totalDelayMS, clockDrift, |
| currentMicLevel); |
| |
| // --- Resample input audio and create/store the initial audio frame |
| GenerateAudioFrame(static_cast<const int16_t*>(audioSamples), |
| nSamples, |
| nChannels, |
| samplesPerSec); |
| |
| // --- Near-end audio processing. |
| ProcessAudio(totalDelayMS, clockDrift, currentMicLevel, keyPressed); |
| |
| if (swap_stereo_channels_ && stereo_codec_) |
| // Only bother swapping if we're using a stereo codec. |
| AudioFrameOperations::SwapStereoChannels(&_audioFrame); |
| |
| // --- Annoying typing detection (utilizes the APM/VAD decision) |
| #if WEBRTC_VOICE_ENGINE_TYPING_DETECTION |
| TypingDetection(keyPressed); |
| #endif |
| |
| // --- Measure audio level of speech after all processing. |
| double sample_duration = static_cast<double>(nSamples) / samplesPerSec; |
| _audioLevel.ComputeLevel(_audioFrame, sample_duration); |
| |
| return 0; |
| } |
| |
| void TransmitMixer::ProcessAndEncodeAudio() { |
| RTC_DCHECK_GT(_audioFrame.samples_per_channel_, 0); |
| for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); |
| it.Increment()) { |
| Channel* const channel = it.GetChannel(); |
| if (channel->Sending()) { |
| channel->ProcessAndEncodeAudio(_audioFrame); |
| } |
| } |
| } |
| |
| uint32_t TransmitMixer::CaptureLevel() const |
| { |
| return _captureLevel; |
| } |
| |
| int32_t |
| TransmitMixer::StopSend() |
| { |
| WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), |
| "TransmitMixer::StopSend()"); |
| _audioLevel.Clear(); |
| return 0; |
| } |
| |
| int8_t TransmitMixer::AudioLevel() const |
| { |
| // Speech + file level [0,9] |
| return _audioLevel.Level(); |
| } |
| |
| int16_t TransmitMixer::AudioLevelFullRange() const |
| { |
| // Speech + file level [0,32767] |
| return _audioLevel.LevelFullRange(); |
| } |
| |
| double TransmitMixer::GetTotalInputEnergy() const { |
| return _audioLevel.TotalEnergy(); |
| } |
| |
| double TransmitMixer::GetTotalInputDuration() const { |
| return _audioLevel.TotalDuration(); |
| } |
| |
| void TransmitMixer::GenerateAudioFrame(const int16_t* audio, |
| size_t samples_per_channel, |
| size_t num_channels, |
| int sample_rate_hz) { |
| int codec_rate; |
| size_t num_codec_channels; |
| GetSendCodecInfo(&codec_rate, &num_codec_channels); |
| stereo_codec_ = num_codec_channels == 2; |
| |
| // We want to process at the lowest rate possible without losing information. |
| // Choose the lowest native rate at least equal to the input and codec rates. |
| const int min_processing_rate = std::min(sample_rate_hz, codec_rate); |
| for (size_t i = 0; i < AudioProcessing::kNumNativeSampleRates; ++i) { |
| _audioFrame.sample_rate_hz_ = AudioProcessing::kNativeSampleRatesHz[i]; |
| if (_audioFrame.sample_rate_hz_ >= min_processing_rate) { |
| break; |
| } |
| } |
| _audioFrame.num_channels_ = std::min(num_channels, num_codec_channels); |
| RemixAndResample(audio, samples_per_channel, num_channels, sample_rate_hz, |
| &resampler_, &_audioFrame); |
| } |
| |
| void TransmitMixer::ProcessAudio(int delay_ms, int clock_drift, |
| int current_mic_level, bool key_pressed) { |
| if (audioproc_->set_stream_delay_ms(delay_ms) != 0) { |
| // Silently ignore this failure to avoid flooding the logs. |
| } |
| |
| GainControl* agc = audioproc_->gain_control(); |
| if (agc->set_stream_analog_level(current_mic_level) != 0) { |
| LOG(LS_ERROR) << "set_stream_analog_level failed: current_mic_level = " |
| << current_mic_level; |
| assert(false); |
| } |
| |
| EchoCancellation* aec = audioproc_->echo_cancellation(); |
| if (aec->is_drift_compensation_enabled()) { |
| aec->set_stream_drift_samples(clock_drift); |
| } |
| |
| audioproc_->set_stream_key_pressed(key_pressed); |
| |
| int err = audioproc_->ProcessStream(&_audioFrame); |
| if (err != 0) { |
| LOG(LS_ERROR) << "ProcessStream() error: " << err; |
| assert(false); |
| } |
| |
| // Store new capture level. Only updated when analog AGC is enabled. |
| _captureLevel = agc->stream_analog_level(); |
| } |
| |
| #if WEBRTC_VOICE_ENGINE_TYPING_DETECTION |
| void TransmitMixer::TypingDetection(bool keyPressed) |
| { |
| // We let the VAD determine if we're using this feature or not. |
| if (_audioFrame.vad_activity_ == AudioFrame::kVadUnknown) { |
| return; |
| } |
| |
| bool vadActive = _audioFrame.vad_activity_ == AudioFrame::kVadActive; |
| if (_typingDetection.Process(keyPressed, vadActive)) { |
| rtc::CritScope cs(&_critSect); |
| _typingNoiseWarningPending = true; |
| _typingNoiseDetected = true; |
| } else { |
| rtc::CritScope cs(&_critSect); |
| // If there is already a warning pending, do not change the state. |
| // Otherwise set a warning pending if last callback was for noise detected. |
| if (!_typingNoiseWarningPending && _typingNoiseDetected) { |
| _typingNoiseWarningPending = true; |
| _typingNoiseDetected = false; |
| } |
| } |
| } |
| #endif |
| |
| void TransmitMixer::EnableStereoChannelSwapping(bool enable) { |
| swap_stereo_channels_ = enable; |
| } |
| |
| bool TransmitMixer::IsStereoChannelSwappingEnabled() { |
| return swap_stereo_channels_; |
| } |
| |
| } // namespace voe |
| } // namespace webrtc |
