Adding 120 ms frame length support in NetEq.
BUG=webrtc:1015
Review-Url: https://codereview.webrtc.org/1901633002
Cr-Commit-Position: refs/heads/master@{#12592}
diff --git a/webrtc/modules/audio_coding/neteq/merge.cc b/webrtc/modules/audio_coding/neteq/merge.cc
index b62df61..94db112 100644
--- a/webrtc/modules/audio_coding/neteq/merge.cc
+++ b/webrtc/modules/audio_coding/neteq/merge.cc
@@ -39,6 +39,8 @@
assert(num_channels_ > 0);
}
+Merge::~Merge() = default;
+
size_t Merge::Process(int16_t* input, size_t input_length,
int16_t* external_mute_factor_array,
AudioMultiVector* output) {
@@ -91,9 +93,8 @@
old_length, input_length_per_channel, expand_period);
}
- static const int kTempDataSize = 3600;
- int16_t temp_data[kTempDataSize]; // TODO(hlundin) Remove this.
- int16_t* decoded_output = temp_data + best_correlation_index;
+ temp_data_.resize(input_length_per_channel + best_correlation_index);
+ int16_t* decoded_output = temp_data_.data() + best_correlation_index;
// Mute the new decoded data if needed (and unmute it linearly).
// This is the overlapping part of expanded_signal.
@@ -127,7 +128,7 @@
int16_t increment =
static_cast<int16_t>(16384 / (interpolation_length + 1)); // In Q14.
int16_t mute_factor = 16384 - increment;
- memmove(temp_data, expanded_channel,
+ memmove(temp_data_.data(), expanded_channel,
sizeof(int16_t) * best_correlation_index);
DspHelper::CrossFade(&expanded_channel[best_correlation_index],
input_channel, interpolation_length,
@@ -140,8 +141,8 @@
} else {
assert(output->Size() == output_length);
}
- memcpy(&(*output)[channel][0], temp_data,
- sizeof(temp_data[0]) * output_length);
+ memcpy(&(*output)[channel][0], temp_data_.data(),
+ sizeof(temp_data_[0]) * output_length);
}
// Copy back the first part of the data to |sync_buffer_| and remove it from
@@ -208,22 +209,20 @@
std::min(static_cast<size_t>(64 * fs_mult_), input_length);
const int16_t expanded_max =
WebRtcSpl_MaxAbsValueW16(expanded_signal, mod_input_length);
- const int16_t input_max = WebRtcSpl_MaxAbsValueW16(input, mod_input_length);
-
- // Calculate energy of expanded signal.
- // |log_fs_mult| is log2(fs_mult_), but is not exact for 48000 Hz.
- int log_fs_mult = 30 - WebRtcSpl_NormW32(fs_mult_);
- int expanded_shift = 6 + log_fs_mult
- - WebRtcSpl_NormW32(expanded_max * expanded_max);
- expanded_shift = std::max(expanded_shift, 0);
+ int32_t factor = (expanded_max * expanded_max) /
+ (std::numeric_limits<int32_t>::max() /
+ static_cast<int32_t>(mod_input_length));
+ const int expanded_shift = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor);
int32_t energy_expanded = WebRtcSpl_DotProductWithScale(expanded_signal,
expanded_signal,
mod_input_length,
expanded_shift);
// Calculate energy of input signal.
- int input_shift = 6 + log_fs_mult - WebRtcSpl_NormW32(input_max * input_max);
- input_shift = std::max(input_shift, 0);
+ const int16_t input_max = WebRtcSpl_MaxAbsValueW16(input, mod_input_length);
+ factor = (input_max * input_max) / (std::numeric_limits<int32_t>::max() /
+ static_cast<int32_t>(mod_input_length));
+ const int input_shift = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor);
int32_t energy_input = WebRtcSpl_DotProductWithScale(input, input,
mod_input_length,
input_shift);
diff --git a/webrtc/modules/audio_coding/neteq/merge.h b/webrtc/modules/audio_coding/neteq/merge.h
index 95dea5a..48f09a1 100644
--- a/webrtc/modules/audio_coding/neteq/merge.h
+++ b/webrtc/modules/audio_coding/neteq/merge.h
@@ -37,7 +37,7 @@
size_t num_channels,
Expand* expand,
SyncBuffer* sync_buffer);
- virtual ~Merge() {}
+ virtual ~Merge();
// The main method to produce the audio data. The decoded data is supplied in
// |input|, having |input_length| samples in total for all channels
@@ -93,6 +93,7 @@
int16_t expanded_downsampled_[kExpandDownsampLength];
int16_t input_downsampled_[kInputDownsampLength];
AudioMultiVector expanded_;
+ std::vector<int16_t> temp_data_;
RTC_DISALLOW_COPY_AND_ASSIGN(Merge);
};
diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc
index ef1e6cb..2eb2277 100644
--- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc
+++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc
@@ -500,6 +500,11 @@
return sync_buffer_.get();
}
+Operations NetEqImpl::last_operation_for_test() const {
+ rtc::CritScope lock(&crit_sect_);
+ return last_operation_;
+}
+
// Methods below this line are private.
int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
@@ -905,6 +910,7 @@
return kInvalidOperation;
}
} // End of switch.
+ last_operation_ = operation;
if (return_value < 0) {
return return_value;
}
diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.h b/webrtc/modules/audio_coding/neteq/neteq_impl.h
index 707fbeb..a707f25 100644
--- a/webrtc/modules/audio_coding/neteq/neteq_impl.h
+++ b/webrtc/modules/audio_coding/neteq/neteq_impl.h
@@ -204,10 +204,11 @@
// This accessor method is only intended for testing purposes.
const SyncBuffer* sync_buffer_for_test() const;
+ Operations last_operation_for_test() const;
protected:
static const int kOutputSizeMs = 10;
- static const size_t kMaxFrameSize = 2880; // 60 ms @ 48 kHz.
+ static const size_t kMaxFrameSize = 5760; // 120 ms @ 48 kHz.
// TODO(hlundin): Provide a better value for kSyncBufferSize.
static const size_t kSyncBufferSize = 2 * kMaxFrameSize;
@@ -383,6 +384,7 @@
size_t output_size_samples_ GUARDED_BY(crit_sect_);
size_t decoder_frame_length_ GUARDED_BY(crit_sect_);
Modes last_mode_ GUARDED_BY(crit_sect_);
+ Operations last_operation_ GUARDED_BY(crit_sect_);
std::unique_ptr<int16_t[]> mute_factor_array_ GUARDED_BY(crit_sect_);
size_t decoded_buffer_length_ GUARDED_BY(crit_sect_);
std::unique_ptr<int16_t[]> decoded_buffer_ GUARDED_BY(crit_sect_);
diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc
index 77622bc..8b47adb 100644
--- a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc
@@ -763,7 +763,7 @@
TEST_F(NetEqImplTest, UnsupportedDecoder) {
UseNoMocks();
CreateInstance();
- static const size_t kNetEqMaxFrameSize = 2880; // 60 ms @ 48 kHz.
+ static const size_t kNetEqMaxFrameSize = 5760; // 120 ms @ 48 kHz.
static const size_t kChannels = 2;
const uint8_t kPayloadType = 17; // Just an arbitrary number.
@@ -773,7 +773,7 @@
const size_t kPayloadLengthSamples =
static_cast<size_t>(10 * kSampleRateHz / 1000); // 10 ms.
const size_t kPayloadLengthBytes = 1;
- uint8_t payload[kPayloadLengthBytes]= {0};
+ uint8_t payload[kPayloadLengthBytes] = {0};
int16_t dummy_output[kPayloadLengthSamples * kChannels] = {0};
WebRtcRTPHeader rtp_header;
rtp_header.header.payloadType = kPayloadType;
@@ -1189,4 +1189,214 @@
EXPECT_EQ(1u, tick_timer_->ticks());
}
+class Decoder120ms : public AudioDecoder {
+ public:
+ Decoder120ms(SpeechType speech_type)
+ : next_value_(1),
+ speech_type_(speech_type) {}
+
+ int DecodeInternal(const uint8_t* encoded,
+ size_t encoded_len,
+ int sample_rate_hz,
+ int16_t* decoded,
+ SpeechType* speech_type) override {
+ size_t decoded_len =
+ rtc::CheckedDivExact(sample_rate_hz, 1000) * 120 * Channels();
+ for (size_t i = 0; i < decoded_len; ++i) {
+ decoded[i] = next_value_++;
+ }
+ *speech_type = speech_type_;
+ return decoded_len;
+ }
+
+ void Reset() override { next_value_ = 1; }
+ size_t Channels() const override { return 2; }
+
+ private:
+ int16_t next_value_;
+ SpeechType speech_type_;
+};
+
+class NetEqImplTest120ms : public NetEqImplTest {
+ protected:
+ NetEqImplTest120ms() : NetEqImplTest() {}
+ virtual ~NetEqImplTest120ms() {}
+
+ void CreateInstanceNoMocks() {
+ UseNoMocks();
+ CreateInstance();
+ }
+
+ void CreateInstanceWithDelayManagerMock() {
+ UseNoMocks();
+ use_mock_delay_manager_ = true;
+ CreateInstance();
+ }
+
+ uint32_t timestamp_diff_between_packets() const {
+ return rtc::CheckedDivExact(kSamplingFreq_, 1000u) * 120;
+ }
+
+ uint32_t first_timestamp() const { return 10u; }
+
+ void GetFirstPacket() {
+ for (int i = 0; i < 12; i++) {
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ }
+ }
+
+ void InsertPacket(uint32_t timestamp) {
+ WebRtcRTPHeader rtp_header;
+ rtp_header.header.payloadType = kPayloadType;
+ rtp_header.header.sequenceNumber = sequence_number_;
+ rtp_header.header.timestamp = timestamp;
+ rtp_header.header.ssrc = 15;
+ const size_t kPayloadLengthBytes = 1; // This can be arbitrary.
+ uint8_t payload[kPayloadLengthBytes] = {0};
+ EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload, 10));
+ sequence_number_++;
+ }
+
+ void Register120msCodec(AudioDecoder::SpeechType speech_type) {
+ decoder_.reset(new Decoder120ms(speech_type));
+ ASSERT_EQ(2u, decoder_->Channels());
+ EXPECT_EQ(NetEq::kOK, neteq_->RegisterExternalDecoder(
+ decoder_.get(), NetEqDecoder::kDecoderOpus_2ch,
+ "120ms codec", kPayloadType, kSamplingFreq_));
+ }
+
+ std::unique_ptr<Decoder120ms> decoder_;
+ AudioFrame output_;
+ const uint32_t kPayloadType = 17;
+ const uint32_t kSamplingFreq_ = 48000;
+ uint16_t sequence_number_ = 1;
+};
+
+TEST_F(NetEqImplTest120ms, AudioRepetition) {
+ config_.playout_mode = kPlayoutFax;
+ CreateInstanceNoMocks();
+ Register120msCodec(AudioDecoder::kSpeech);
+
+ InsertPacket(first_timestamp());
+ GetFirstPacket();
+
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ EXPECT_EQ(kAudioRepetition, neteq_->last_operation_for_test());
+}
+
+TEST_F(NetEqImplTest120ms, AlternativePlc) {
+ config_.playout_mode = kPlayoutOff;
+ CreateInstanceNoMocks();
+ Register120msCodec(AudioDecoder::kSpeech);
+
+ InsertPacket(first_timestamp());
+ GetFirstPacket();
+
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ EXPECT_EQ(kAlternativePlc, neteq_->last_operation_for_test());
+}
+
+TEST_F(NetEqImplTest120ms, CodecInternalCng) {
+ CreateInstanceNoMocks();
+ Register120msCodec(AudioDecoder::kComfortNoise);
+
+ InsertPacket(first_timestamp());
+ GetFirstPacket();
+
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ EXPECT_EQ(kCodecInternalCng, neteq_->last_operation_for_test());
+}
+
+TEST_F(NetEqImplTest120ms, Normal) {
+ CreateInstanceNoMocks();
+ Register120msCodec(AudioDecoder::kSpeech);
+
+ InsertPacket(first_timestamp());
+ GetFirstPacket();
+
+ EXPECT_EQ(kNormal, neteq_->last_operation_for_test());
+}
+
+TEST_F(NetEqImplTest120ms, Merge) {
+ CreateInstanceWithDelayManagerMock();
+
+ Register120msCodec(AudioDecoder::kSpeech);
+ InsertPacket(first_timestamp());
+
+ GetFirstPacket();
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+
+ InsertPacket(first_timestamp() + 2 * timestamp_diff_between_packets());
+
+ // Delay manager reports a target level which should cause a Merge.
+ EXPECT_CALL(*mock_delay_manager_, TargetLevel()).WillOnce(Return(-10));
+
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ EXPECT_EQ(kMerge, neteq_->last_operation_for_test());
+}
+
+TEST_F(NetEqImplTest120ms, Expand) {
+ CreateInstanceNoMocks();
+ Register120msCodec(AudioDecoder::kSpeech);
+
+ InsertPacket(first_timestamp());
+ GetFirstPacket();
+
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ EXPECT_EQ(kExpand, neteq_->last_operation_for_test());
+}
+
+TEST_F(NetEqImplTest120ms, FastAccelerate) {
+ CreateInstanceWithDelayManagerMock();
+ Register120msCodec(AudioDecoder::kSpeech);
+
+ InsertPacket(first_timestamp());
+ GetFirstPacket();
+ InsertPacket(first_timestamp() + timestamp_diff_between_packets());
+
+ // Delay manager report buffer limit which should cause a FastAccelerate.
+ EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _))
+ .Times(1)
+ .WillOnce(DoAll(SetArgPointee<0>(0), SetArgPointee<1>(0)));
+
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ EXPECT_EQ(kFastAccelerate, neteq_->last_operation_for_test());
+}
+
+TEST_F(NetEqImplTest120ms, PreemptiveExpand) {
+ CreateInstanceWithDelayManagerMock();
+ Register120msCodec(AudioDecoder::kSpeech);
+
+ InsertPacket(first_timestamp());
+ GetFirstPacket();
+
+ InsertPacket(first_timestamp() + timestamp_diff_between_packets());
+
+ // Delay manager report buffer limit which should cause a PreemptiveExpand.
+ EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _))
+ .Times(1)
+ .WillOnce(DoAll(SetArgPointee<0>(100), SetArgPointee<1>(100)));
+
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ EXPECT_EQ(kPreemptiveExpand, neteq_->last_operation_for_test());
+}
+
+TEST_F(NetEqImplTest120ms, Accelerate) {
+ CreateInstanceWithDelayManagerMock();
+ Register120msCodec(AudioDecoder::kSpeech);
+
+ InsertPacket(first_timestamp());
+ GetFirstPacket();
+
+ InsertPacket(first_timestamp() + timestamp_diff_between_packets());
+
+ // Delay manager report buffer limit which should cause a Accelerate.
+ EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _))
+ .Times(1)
+ .WillOnce(DoAll(SetArgPointee<0>(1), SetArgPointee<1>(2)));
+
+ EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_));
+ EXPECT_EQ(kAccelerate, neteq_->last_operation_for_test());
+}
+
}// namespace webrtc
diff --git a/webrtc/modules/audio_coding/neteq/normal_unittest.cc b/webrtc/modules/audio_coding/neteq/normal_unittest.cc
index f98e99a..5e1fc13 100644
--- a/webrtc/modules/audio_coding/neteq/normal_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/normal_unittest.cc
@@ -27,9 +27,20 @@
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
using ::testing::_;
+using ::testing::Invoke;
namespace webrtc {
+namespace {
+
+int ExpandProcess120ms(AudioMultiVector* output) {
+ AudioMultiVector dummy_audio(1, 11520u);
+ dummy_audio.CopyTo(output);
+ return 0;
+}
+
+} // namespace
+
TEST(Normal, CreateAndDestroy) {
MockDecoderDatabase db;
int fs = 8000;
@@ -121,6 +132,45 @@
EXPECT_CALL(expand, Die()); // Called when |expand| goes out of scope.
}
+TEST(Normal, LastModeExpand120msPacket) {
+ WebRtcSpl_Init();
+ MockDecoderDatabase db;
+ const int kFs = 48000;
+ const size_t kPacketsizeBytes = 11520u;
+ const size_t kChannels = 1;
+ BackgroundNoise bgn(kChannels);
+ SyncBuffer sync_buffer(kChannels, 1000);
+ RandomVector random_vector;
+ StatisticsCalculator statistics;
+ MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, kFs,
+ kChannels);
+ Normal normal(kFs, &db, bgn, &expand);
+
+ int16_t input[kPacketsizeBytes] = {0};
+
+ std::unique_ptr<int16_t[]> mute_factor_array(new int16_t[kChannels]);
+ for (size_t i = 0; i < kChannels; ++i) {
+ mute_factor_array[i] = 16384;
+ }
+
+ AudioMultiVector output(kChannels);
+
+ EXPECT_CALL(expand, SetParametersForNormalAfterExpand());
+ EXPECT_CALL(expand, Process(_)).WillOnce(Invoke(ExpandProcess120ms));
+ EXPECT_CALL(expand, Reset());
+ EXPECT_EQ(static_cast<int>(kPacketsizeBytes),
+ normal.Process(input,
+ kPacketsizeBytes,
+ kModeExpand,
+ mute_factor_array.get(),
+ &output));
+
+ EXPECT_EQ(kPacketsizeBytes, output.Size());
+
+ EXPECT_CALL(db, Die()); // Called when |db| goes out of scope.
+ EXPECT_CALL(expand, Die()); // Called when |expand| goes out of scope.
+}
+
// TODO(hlundin): Write more tests.
} // namespace webrtc
