blob: 4e0c26dbf08cb68519206a0ebf9e6a47f63e05dc [file] [log] [blame]
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_device/android/opensles_recorder.h"
#include <android/log.h>
#include <memory>
#include "api/array_view.h"
#include "modules/audio_device/android/audio_common.h"
#include "modules/audio_device/android/audio_manager.h"
#include "modules/audio_device/fine_audio_buffer.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/platform_thread.h"
#include "rtc_base/time_utils.h"
#define TAG "OpenSLESRecorder"
#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
#define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__)
#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
#define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
#define LOG_ON_ERROR(op) \
[](SLresult err) { \
if (err != SL_RESULT_SUCCESS) { \
ALOGE("%s:%d %s failed: %s", __FILE__, __LINE__, #op, \
GetSLErrorString(err)); \
return true; \
} \
return false; \
}(op)
namespace webrtc {
OpenSLESRecorder::OpenSLESRecorder(AudioManager* audio_manager)
: audio_manager_(audio_manager),
audio_parameters_(audio_manager->GetRecordAudioParameters()),
audio_device_buffer_(nullptr),
initialized_(false),
recording_(false),
engine_(nullptr),
recorder_(nullptr),
simple_buffer_queue_(nullptr),
buffer_index_(0),
last_rec_time_(0) {
ALOGD("ctor[tid=%d]", rtc::CurrentThreadId());
// Detach from this thread since we want to use the checker to verify calls
// from the internal audio thread.
thread_checker_opensles_.Detach();
// Use native audio output parameters provided by the audio manager and
// define the PCM format structure.
pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(),
audio_parameters_.sample_rate(),
audio_parameters_.bits_per_sample());
}
OpenSLESRecorder::~OpenSLESRecorder() {
ALOGD("dtor[tid=%d]", rtc::CurrentThreadId());
RTC_DCHECK(thread_checker_.IsCurrent());
Terminate();
DestroyAudioRecorder();
engine_ = nullptr;
RTC_DCHECK(!engine_);
RTC_DCHECK(!recorder_);
RTC_DCHECK(!simple_buffer_queue_);
}
int OpenSLESRecorder::Init() {
ALOGD("Init[tid=%d]", rtc::CurrentThreadId());
RTC_DCHECK(thread_checker_.IsCurrent());
if (audio_parameters_.channels() == 2) {
ALOGD("Stereo mode is enabled");
}
return 0;
}
int OpenSLESRecorder::Terminate() {
ALOGD("Terminate[tid=%d]", rtc::CurrentThreadId());
RTC_DCHECK(thread_checker_.IsCurrent());
StopRecording();
return 0;
}
int OpenSLESRecorder::InitRecording() {
ALOGD("InitRecording[tid=%d]", rtc::CurrentThreadId());
RTC_DCHECK(thread_checker_.IsCurrent());
RTC_DCHECK(!initialized_);
RTC_DCHECK(!recording_);
if (!ObtainEngineInterface()) {
ALOGE("Failed to obtain SL Engine interface");
return -1;
}
CreateAudioRecorder();
initialized_ = true;
buffer_index_ = 0;
return 0;
}
int OpenSLESRecorder::StartRecording() {
ALOGD("StartRecording[tid=%d]", rtc::CurrentThreadId());
RTC_DCHECK(thread_checker_.IsCurrent());
RTC_DCHECK(initialized_);
RTC_DCHECK(!recording_);
if (fine_audio_buffer_) {
fine_audio_buffer_->ResetRecord();
}
// Add buffers to the queue before changing state to SL_RECORDSTATE_RECORDING
// to ensure that recording starts as soon as the state is modified. On some
// devices, SLAndroidSimpleBufferQueue::Clear() used in Stop() does not flush
// the buffers as intended and we therefore check the number of buffers
// already queued first. Enqueue() can return SL_RESULT_BUFFER_INSUFFICIENT
// otherwise.
int num_buffers_in_queue = GetBufferCount();
for (int i = 0; i < kNumOfOpenSLESBuffers - num_buffers_in_queue; ++i) {
if (!EnqueueAudioBuffer()) {
recording_ = false;
return -1;
}
}
num_buffers_in_queue = GetBufferCount();
RTC_DCHECK_EQ(num_buffers_in_queue, kNumOfOpenSLESBuffers);
LogBufferState();
// Start audio recording by changing the state to SL_RECORDSTATE_RECORDING.
// Given that buffers are already enqueued, recording should start at once.
// The macro returns -1 if recording fails to start.
last_rec_time_ = rtc::Time();
if (LOG_ON_ERROR(
(*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_RECORDING))) {
return -1;
}
recording_ = (GetRecordState() == SL_RECORDSTATE_RECORDING);
RTC_DCHECK(recording_);
return 0;
}
int OpenSLESRecorder::StopRecording() {
ALOGD("StopRecording[tid=%d]", rtc::CurrentThreadId());
RTC_DCHECK(thread_checker_.IsCurrent());
if (!initialized_ || !recording_) {
return 0;
}
// Stop recording by setting the record state to SL_RECORDSTATE_STOPPED.
if (LOG_ON_ERROR(
(*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_STOPPED))) {
return -1;
}
// Clear the buffer queue to get rid of old data when resuming recording.
if (LOG_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_))) {
return -1;
}
thread_checker_opensles_.Detach();
initialized_ = false;
recording_ = false;
return 0;
}
void OpenSLESRecorder::AttachAudioBuffer(AudioDeviceBuffer* audio_buffer) {
ALOGD("AttachAudioBuffer");
RTC_DCHECK(thread_checker_.IsCurrent());
RTC_CHECK(audio_buffer);
audio_device_buffer_ = audio_buffer;
// Ensure that the audio device buffer is informed about the native sample
// rate used on the recording side.
const int sample_rate_hz = audio_parameters_.sample_rate();
ALOGD("SetRecordingSampleRate(%d)", sample_rate_hz);
audio_device_buffer_->SetRecordingSampleRate(sample_rate_hz);
// Ensure that the audio device buffer is informed about the number of
// channels preferred by the OS on the recording side.
const size_t channels = audio_parameters_.channels();
ALOGD("SetRecordingChannels(%zu)", channels);
audio_device_buffer_->SetRecordingChannels(channels);
// Allocated memory for internal data buffers given existing audio parameters.
AllocateDataBuffers();
}
int OpenSLESRecorder::EnableBuiltInAEC(bool enable) {
ALOGD("EnableBuiltInAEC(%d)", enable);
RTC_DCHECK(thread_checker_.IsCurrent());
ALOGE("Not implemented");
return 0;
}
int OpenSLESRecorder::EnableBuiltInAGC(bool enable) {
ALOGD("EnableBuiltInAGC(%d)", enable);
RTC_DCHECK(thread_checker_.IsCurrent());
ALOGE("Not implemented");
return 0;
}
int OpenSLESRecorder::EnableBuiltInNS(bool enable) {
ALOGD("EnableBuiltInNS(%d)", enable);
RTC_DCHECK(thread_checker_.IsCurrent());
ALOGE("Not implemented");
return 0;
}
bool OpenSLESRecorder::ObtainEngineInterface() {
ALOGD("ObtainEngineInterface");
RTC_DCHECK(thread_checker_.IsCurrent());
if (engine_)
return true;
// Get access to (or create if not already existing) the global OpenSL Engine
// object.
SLObjectItf engine_object = audio_manager_->GetOpenSLEngine();
if (engine_object == nullptr) {
ALOGE("Failed to access the global OpenSL engine");
return false;
}
// Get the SL Engine Interface which is implicit.
if (LOG_ON_ERROR(
(*engine_object)
->GetInterface(engine_object, SL_IID_ENGINE, &engine_))) {
return false;
}
return true;
}
bool OpenSLESRecorder::CreateAudioRecorder() {
ALOGD("CreateAudioRecorder");
RTC_DCHECK(thread_checker_.IsCurrent());
if (recorder_object_.Get())
return true;
RTC_DCHECK(!recorder_);
RTC_DCHECK(!simple_buffer_queue_);
// Audio source configuration.
SLDataLocator_IODevice mic_locator = {SL_DATALOCATOR_IODEVICE,
SL_IODEVICE_AUDIOINPUT,
SL_DEFAULTDEVICEID_AUDIOINPUT, NULL};
SLDataSource audio_source = {&mic_locator, NULL};
// Audio sink configuration.
SLDataLocator_AndroidSimpleBufferQueue buffer_queue = {
SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
static_cast<SLuint32>(kNumOfOpenSLESBuffers)};
SLDataSink audio_sink = {&buffer_queue, &pcm_format_};
// Create the audio recorder object (requires the RECORD_AUDIO permission).
// Do not realize the recorder yet. Set the configuration first.
const SLInterfaceID interface_id[] = {SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
SL_IID_ANDROIDCONFIGURATION};
const SLboolean interface_required[] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE};
if (LOG_ON_ERROR((*engine_)->CreateAudioRecorder(
engine_, recorder_object_.Receive(), &audio_source, &audio_sink,
arraysize(interface_id), interface_id, interface_required))) {
return false;
}
// Configure the audio recorder (before it is realized).
SLAndroidConfigurationItf recorder_config;
if (LOG_ON_ERROR((recorder_object_->GetInterface(recorder_object_.Get(),
SL_IID_ANDROIDCONFIGURATION,
&recorder_config)))) {
return false;
}
// Uses the default microphone tuned for audio communication.
// Note that, SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION leads to a fast
// track but also excludes usage of required effects like AEC, AGC and NS.
// SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION
SLint32 stream_type = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION;
if (LOG_ON_ERROR(((*recorder_config)
->SetConfiguration(recorder_config,
SL_ANDROID_KEY_RECORDING_PRESET,
&stream_type, sizeof(SLint32))))) {
return false;
}
// The audio recorder can now be realized (in synchronous mode).
if (LOG_ON_ERROR((recorder_object_->Realize(recorder_object_.Get(),
SL_BOOLEAN_FALSE)))) {
return false;
}
// Get the implicit recorder interface (SL_IID_RECORD).
if (LOG_ON_ERROR((recorder_object_->GetInterface(
recorder_object_.Get(), SL_IID_RECORD, &recorder_)))) {
return false;
}
// Get the simple buffer queue interface (SL_IID_ANDROIDSIMPLEBUFFERQUEUE).
// It was explicitly requested.
if (LOG_ON_ERROR((recorder_object_->GetInterface(
recorder_object_.Get(), SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
&simple_buffer_queue_)))) {
return false;
}
// Register the input callback for the simple buffer queue.
// This callback will be called when receiving new data from the device.
if (LOG_ON_ERROR(((*simple_buffer_queue_)
->RegisterCallback(simple_buffer_queue_,
SimpleBufferQueueCallback, this)))) {
return false;
}
return true;
}
void OpenSLESRecorder::DestroyAudioRecorder() {
ALOGD("DestroyAudioRecorder");
RTC_DCHECK(thread_checker_.IsCurrent());
if (!recorder_object_.Get())
return;
(*simple_buffer_queue_)
->RegisterCallback(simple_buffer_queue_, nullptr, nullptr);
recorder_object_.Reset();
recorder_ = nullptr;
simple_buffer_queue_ = nullptr;
}
void OpenSLESRecorder::SimpleBufferQueueCallback(
SLAndroidSimpleBufferQueueItf buffer_queue,
void* context) {
OpenSLESRecorder* stream = static_cast<OpenSLESRecorder*>(context);
stream->ReadBufferQueue();
}
void OpenSLESRecorder::AllocateDataBuffers() {
ALOGD("AllocateDataBuffers");
RTC_DCHECK(thread_checker_.IsCurrent());
RTC_DCHECK(!simple_buffer_queue_);
RTC_CHECK(audio_device_buffer_);
// Create a modified audio buffer class which allows us to deliver any number
// of samples (and not only multiple of 10ms) to match the native audio unit
// buffer size.
ALOGD("frames per native buffer: %zu", audio_parameters_.frames_per_buffer());
ALOGD("frames per 10ms buffer: %zu",
audio_parameters_.frames_per_10ms_buffer());
ALOGD("bytes per native buffer: %zu", audio_parameters_.GetBytesPerBuffer());
ALOGD("native sample rate: %d", audio_parameters_.sample_rate());
RTC_DCHECK(audio_device_buffer_);
fine_audio_buffer_ = std::make_unique<FineAudioBuffer>(audio_device_buffer_);
// Allocate queue of audio buffers that stores recorded audio samples.
const int buffer_size_samples =
audio_parameters_.frames_per_buffer() * audio_parameters_.channels();
audio_buffers_.reset(new std::unique_ptr<SLint16[]>[kNumOfOpenSLESBuffers]);
for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
audio_buffers_[i].reset(new SLint16[buffer_size_samples]);
}
}
void OpenSLESRecorder::ReadBufferQueue() {
RTC_DCHECK(thread_checker_opensles_.IsCurrent());
SLuint32 state = GetRecordState();
if (state != SL_RECORDSTATE_RECORDING) {
ALOGW("Buffer callback in non-recording state!");
return;
}
// Check delta time between two successive callbacks and provide a warning
// if it becomes very large.
// TODO(henrika): using 150ms as upper limit but this value is rather random.
const uint32_t current_time = rtc::Time();
const uint32_t diff = current_time - last_rec_time_;
if (diff > 150) {
ALOGW("Bad OpenSL ES record timing, dT=%u [ms]", diff);
}
last_rec_time_ = current_time;
// Send recorded audio data to the WebRTC sink.
// TODO(henrika): fix delay estimates. It is OK to use fixed values for now
// since there is no support to turn off built-in EC in combination with
// OpenSL ES anyhow. Hence, as is, the WebRTC based AEC (which would use
// these estimates) will never be active.
fine_audio_buffer_->DeliverRecordedData(
rtc::ArrayView<const int16_t>(
audio_buffers_[buffer_index_].get(),
audio_parameters_.frames_per_buffer() * audio_parameters_.channels()),
25);
// Enqueue the utilized audio buffer and use if for recording again.
EnqueueAudioBuffer();
}
bool OpenSLESRecorder::EnqueueAudioBuffer() {
SLresult err =
(*simple_buffer_queue_)
->Enqueue(
simple_buffer_queue_,
reinterpret_cast<SLint8*>(audio_buffers_[buffer_index_].get()),
audio_parameters_.GetBytesPerBuffer());
if (SL_RESULT_SUCCESS != err) {
ALOGE("Enqueue failed: %s", GetSLErrorString(err));
return false;
}
buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers;
return true;
}
SLuint32 OpenSLESRecorder::GetRecordState() const {
RTC_DCHECK(recorder_);
SLuint32 state;
SLresult err = (*recorder_)->GetRecordState(recorder_, &state);
if (SL_RESULT_SUCCESS != err) {
ALOGE("GetRecordState failed: %s", GetSLErrorString(err));
}
return state;
}
SLAndroidSimpleBufferQueueState OpenSLESRecorder::GetBufferQueueState() const {
RTC_DCHECK(simple_buffer_queue_);
// state.count: Number of buffers currently in the queue.
// state.index: Index of the currently filling buffer. This is a linear index
// that keeps a cumulative count of the number of buffers recorded.
SLAndroidSimpleBufferQueueState state;
SLresult err =
(*simple_buffer_queue_)->GetState(simple_buffer_queue_, &state);
if (SL_RESULT_SUCCESS != err) {
ALOGE("GetState failed: %s", GetSLErrorString(err));
}
return state;
}
void OpenSLESRecorder::LogBufferState() const {
SLAndroidSimpleBufferQueueState state = GetBufferQueueState();
ALOGD("state.count:%d state.index:%d", state.count, state.index);
}
SLuint32 OpenSLESRecorder::GetBufferCount() {
SLAndroidSimpleBufferQueueState state = GetBufferQueueState();
return state.count;
}
} // namespace webrtc