Moved Opus-specific payload splitting into AudioDecoderOpus.
The biggest change to NetEq is the move from a primary flag, to a
Priority with two separate levels: one set by RED splitting and one
set by the codec itself. This allows us to unambigously prioritize
"fallback" packets from these two sources. I've chosen what I believe
is the sensible ordering: packets that the codec prioritizes are
chosen first, regardless of if they are secondary RED packets or
not. So if we were to use Opus w/ FEC in RED, we'd only do Opus FEC
decoding if there was no RED packet that could cover the time slot.
With this change, PayloadSplitter now only deals with RED
packets. Maybe it should be renamed RedPayloadSplitter?
BUG=webrtc:5805
Review-Url: https://codereview.webrtc.org/2342443005
Cr-Commit-Position: refs/heads/master@{#14347}
diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn
index 8fb9f86..11a5ef0 100644
--- a/webrtc/modules/BUILD.gn
+++ b/webrtc/modules/BUILD.gn
@@ -313,7 +313,7 @@
"audio_coding/neteq/mock/mock_expand.h",
"audio_coding/neteq/mock/mock_external_decoder_pcm16b.h",
"audio_coding/neteq/mock/mock_packet_buffer.h",
- "audio_coding/neteq/mock/mock_payload_splitter.h",
+ "audio_coding/neteq/mock/mock_red_payload_splitter.h",
"audio_coding/neteq/nack_tracker_unittest.cc",
"audio_coding/neteq/neteq_external_decoder_unittest.cc",
"audio_coding/neteq/neteq_impl_unittest.cc",
@@ -322,9 +322,9 @@
"audio_coding/neteq/neteq_unittest.cc",
"audio_coding/neteq/normal_unittest.cc",
"audio_coding/neteq/packet_buffer_unittest.cc",
- "audio_coding/neteq/payload_splitter_unittest.cc",
"audio_coding/neteq/post_decode_vad_unittest.cc",
"audio_coding/neteq/random_vector_unittest.cc",
+ "audio_coding/neteq/red_payload_splitter_unittest.cc",
"audio_coding/neteq/sync_buffer_unittest.cc",
"audio_coding/neteq/tick_timer_unittest.cc",
"audio_coding/neteq/time_stretch_unittest.cc",
diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn
index a60baba..b55829d 100644
--- a/webrtc/modules/audio_coding/BUILD.gn
+++ b/webrtc/modules/audio_coding/BUILD.gn
@@ -784,14 +784,14 @@
"neteq/packet.h",
"neteq/packet_buffer.cc",
"neteq/packet_buffer.h",
- "neteq/payload_splitter.cc",
- "neteq/payload_splitter.h",
"neteq/post_decode_vad.cc",
"neteq/post_decode_vad.h",
"neteq/preemptive_expand.cc",
"neteq/preemptive_expand.h",
"neteq/random_vector.cc",
"neteq/random_vector.h",
+ "neteq/red_payload_splitter.cc",
+ "neteq/red_payload_splitter.h",
"neteq/rtcp.cc",
"neteq/rtcp.h",
"neteq/statistics_calculator.cc",
diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.cc b/webrtc/modules/audio_coding/codecs/audio_decoder.cc
index 6c67924..afa5115 100644
--- a/webrtc/modules/audio_coding/codecs/audio_decoder.cc
+++ b/webrtc/modules/audio_coding/codecs/audio_decoder.cc
@@ -14,8 +14,6 @@
#include <memory>
#include <utility>
-#include <utility>
-
#include "webrtc/base/array_view.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/sanitizer.h"
@@ -27,9 +25,11 @@
AudioDecoder::ParseResult::ParseResult() = default;
AudioDecoder::ParseResult::ParseResult(ParseResult&& b) = default;
AudioDecoder::ParseResult::ParseResult(uint32_t timestamp,
- bool primary,
+ int priority,
std::unique_ptr<EncodedAudioFrame> frame)
- : timestamp(timestamp), primary(primary), frame(std::move(frame)) {}
+ : timestamp(timestamp), priority(priority), frame(std::move(frame)) {
+ RTC_DCHECK_GE(priority, 0);
+}
AudioDecoder::ParseResult::~ParseResult() = default;
@@ -38,12 +38,11 @@
std::vector<AudioDecoder::ParseResult> AudioDecoder::ParsePayload(
rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) {
+ uint32_t timestamp) {
std::vector<ParseResult> results;
std::unique_ptr<EncodedAudioFrame> frame(
- new LegacyEncodedAudioFrame(this, std::move(payload), is_primary));
- results.emplace_back(timestamp, is_primary, std::move(frame));
+ new LegacyEncodedAudioFrame(this, std::move(payload)));
+ results.emplace_back(timestamp, 0, std::move(frame));
return results;
}
diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.h b/webrtc/modules/audio_coding/codecs/audio_decoder.h
index b6338d2..8468da2 100644
--- a/webrtc/modules/audio_coding/codecs/audio_decoder.h
+++ b/webrtc/modules/audio_coding/codecs/audio_decoder.h
@@ -8,11 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_
-#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_
-
-#include <memory>
-#include <vector>
+#ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_
+#define WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_
#include <memory>
#include <vector>
@@ -66,7 +63,7 @@
struct ParseResult {
ParseResult();
ParseResult(uint32_t timestamp,
- bool primary,
+ int priority,
std::unique_ptr<EncodedAudioFrame> frame);
ParseResult(ParseResult&& b);
~ParseResult();
@@ -75,7 +72,10 @@
// The timestamp of the frame is in samples per channel.
uint32_t timestamp;
- bool primary;
+ // The relative priority of the frame compared to other frames of the same
+ // payload and the same timeframe. A higher value means a lower priority.
+ // The highest priority is zero - negative values are not allowed.
+ int priority;
std::unique_ptr<EncodedAudioFrame> frame;
};
@@ -86,8 +86,7 @@
// buffer. |timestamp| is the input timestamp, in samples, corresponding to
// the start of the payload.
virtual std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary);
+ uint32_t timestamp);
// Decodes |encode_len| bytes from |encoded| and writes the result in
// |decoded|. The maximum bytes allowed to be written into |decoded| is
@@ -177,4 +176,4 @@
};
} // namespace webrtc
-#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_AUDIO_DECODER_H_
+#endif // WEBRTC_MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_
diff --git a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc
index f2fdb1f..fd285a7 100644
--- a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc
+++ b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc
@@ -19,10 +19,9 @@
std::vector<AudioDecoder::ParseResult> AudioDecoderPcmU::ParsePayload(
rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) {
+ uint32_t timestamp) {
return LegacyEncodedAudioFrame::SplitBySamples(
- this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8);
+ this, std::move(payload), timestamp, 8 * num_channels_, 8);
}
int AudioDecoderPcmU::SampleRateHz() const {
@@ -55,10 +54,9 @@
std::vector<AudioDecoder::ParseResult> AudioDecoderPcmA::ParsePayload(
rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) {
+ uint32_t timestamp) {
return LegacyEncodedAudioFrame::SplitBySamples(
- this, std::move(payload), timestamp, is_primary, 8 * num_channels_, 8);
+ this, std::move(payload), timestamp, 8 * num_channels_, 8);
}
int AudioDecoderPcmA::SampleRateHz() const {
diff --git a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h
index ed39021..483311b 100644
--- a/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h
+++ b/webrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h
@@ -24,8 +24,7 @@
}
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) override;
+ uint32_t timestamp) override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int SampleRateHz() const override;
size_t Channels() const override;
@@ -49,8 +48,7 @@
}
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) override;
+ uint32_t timestamp) override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int SampleRateHz() const override;
size_t Channels() const override;
diff --git a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc
index 93b24bd..3a0f6ed 100644
--- a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc
+++ b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc
@@ -50,10 +50,9 @@
std::vector<AudioDecoder::ParseResult> AudioDecoderG722::ParsePayload(
rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) {
+ uint32_t timestamp) {
return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload),
- timestamp, is_primary, 8, 16);
+ timestamp, 8, 16);
}
int AudioDecoderG722::PacketDuration(const uint8_t* encoded,
@@ -128,10 +127,9 @@
std::vector<AudioDecoder::ParseResult> AudioDecoderG722Stereo::ParsePayload(
rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) {
- return LegacyEncodedAudioFrame::SplitBySamples(
- this, std::move(payload), timestamp, is_primary, 2 * 8, 16);
+ uint32_t timestamp) {
+ return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload),
+ timestamp, 2 * 8, 16);
}
// Split the stereo packet and place left and right channel after each other
diff --git a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h
index ad39619..9e7c5d9 100644
--- a/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h
+++ b/webrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h
@@ -25,8 +25,7 @@
bool HasDecodePlc() const override;
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) override;
+ uint32_t timestamp) override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int SampleRateHz() const override;
size_t Channels() const override;
@@ -49,8 +48,7 @@
~AudioDecoderG722Stereo() override;
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) override;
+ uint32_t timestamp) override;
int SampleRateHz() const override;
size_t Channels() const override;
diff --git a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc
index b4bd599..354f819 100644
--- a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc
+++ b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc
@@ -10,6 +10,8 @@
#include "webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h"
+#include <utility>
+
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/modules/audio_coding/codecs/ilbc/ilbc.h"
@@ -53,8 +55,7 @@
std::vector<AudioDecoder::ParseResult> AudioDecoderIlbc::ParsePayload(
rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) {
+ uint32_t timestamp) {
std::vector<ParseResult> results;
size_t bytes_per_frame;
int timestamps_per_frame;
@@ -78,8 +79,8 @@
RTC_DCHECK_EQ(0u, payload.size() % bytes_per_frame);
if (payload.size() == bytes_per_frame) {
std::unique_ptr<EncodedAudioFrame> frame(
- new LegacyEncodedAudioFrame(this, std::move(payload), is_primary));
- results.emplace_back(timestamp, is_primary, std::move(frame));
+ new LegacyEncodedAudioFrame(this, std::move(payload)));
+ results.emplace_back(timestamp, 0, std::move(frame));
} else {
size_t byte_offset;
uint32_t timestamp_offset;
@@ -87,11 +88,9 @@
byte_offset < payload.size();
byte_offset += bytes_per_frame,
timestamp_offset += timestamps_per_frame) {
- rtc::Buffer new_payload(payload.data() + byte_offset, bytes_per_frame);
std::unique_ptr<EncodedAudioFrame> frame(new LegacyEncodedAudioFrame(
- this, std::move(new_payload), is_primary));
- results.emplace_back(timestamp + timestamp_offset, is_primary,
- std::move(frame));
+ this, rtc::Buffer(payload.data() + byte_offset, bytes_per_frame)));
+ results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame));
}
}
diff --git a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h
index fdc856e..48edfed 100644
--- a/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h
+++ b/webrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h
@@ -26,8 +26,7 @@
size_t DecodePlc(size_t num_frames, int16_t* decoded) override;
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) override;
+ uint32_t timestamp) override;
int SampleRateHz() const override;
size_t Channels() const override;
diff --git a/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc b/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc
index a8b76a5..2abdaca 100644
--- a/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc
+++ b/webrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc
@@ -81,7 +81,7 @@
};
const auto results = decoder.ParsePayload(
- generate_payload(frame_length_bytes_ * num_frames_), 0, true);
+ generate_payload(frame_length_bytes_ * num_frames_), 0);
EXPECT_EQ(num_frames_, results.size());
size_t frame_num = 0;
@@ -123,7 +123,7 @@
AudioDecoderIlbc decoder;
constexpr size_t kPayloadLengthBytes = 950;
const auto results =
- decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true);
+ decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0);
EXPECT_TRUE(results.empty());
}
@@ -132,7 +132,7 @@
AudioDecoderIlbc decoder;
constexpr size_t kPayloadLengthBytes = 39; // Not an even number of frames.
const auto results =
- decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0, true);
+ decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0);
EXPECT_TRUE(results.empty());
}
diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc
index 5e6ff01..e0f1faf 100644
--- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc
+++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc
@@ -17,37 +17,22 @@
namespace webrtc {
LegacyEncodedAudioFrame::LegacyEncodedAudioFrame(AudioDecoder* decoder,
- rtc::Buffer&& payload,
- bool is_primary_payload)
- : decoder_(decoder),
- payload_(std::move(payload)),
- is_primary_payload_(is_primary_payload) {}
+ rtc::Buffer&& payload)
+ : decoder_(decoder), payload_(std::move(payload)) {}
LegacyEncodedAudioFrame::~LegacyEncodedAudioFrame() = default;
size_t LegacyEncodedAudioFrame::Duration() const {
- int ret;
- if (is_primary_payload_) {
- ret = decoder_->PacketDuration(payload_.data(), payload_.size());
- } else {
- ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size());
- }
+ const int ret = decoder_->PacketDuration(payload_.data(), payload_.size());
return (ret < 0) ? 0 : static_cast<size_t>(ret);
}
rtc::Optional<AudioDecoder::EncodedAudioFrame::DecodeResult>
LegacyEncodedAudioFrame::Decode(rtc::ArrayView<int16_t> decoded) const {
AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech;
- int ret;
- if (is_primary_payload_) {
- ret = decoder_->Decode(
- payload_.data(), payload_.size(), decoder_->SampleRateHz(),
- decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
- } else {
- ret = decoder_->DecodeRedundant(
- payload_.data(), payload_.size(), decoder_->SampleRateHz(),
- decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
- }
+ const int ret = decoder_->Decode(
+ payload_.data(), payload_.size(), decoder_->SampleRateHz(),
+ decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
if (ret < 0)
return rtc::Optional<DecodeResult>();
@@ -59,7 +44,6 @@
AudioDecoder* decoder,
rtc::Buffer&& payload,
uint32_t timestamp,
- bool is_primary,
size_t bytes_per_ms,
uint32_t timestamps_per_ms) {
RTC_DCHECK(payload.data());
@@ -70,8 +54,8 @@
const size_t min_chunk_size = bytes_per_ms * 20;
if (min_chunk_size >= payload.size()) {
std::unique_ptr<LegacyEncodedAudioFrame> frame(
- new LegacyEncodedAudioFrame(decoder, std::move(payload), is_primary));
- results.emplace_back(timestamp, is_primary, std::move(frame));
+ new LegacyEncodedAudioFrame(decoder, std::move(payload)));
+ results.emplace_back(timestamp, 0, std::move(frame));
} else {
// Reduce the split size by half as long as |split_size_bytes| is at least
// twice the minimum chunk size (so that the resulting size is at least as
@@ -92,10 +76,8 @@
std::min(split_size_bytes, payload.size() - byte_offset);
rtc::Buffer new_payload(payload.data() + byte_offset, split_size_bytes);
std::unique_ptr<LegacyEncodedAudioFrame> frame(
- new LegacyEncodedAudioFrame(decoder, std::move(new_payload),
- is_primary));
- results.emplace_back(timestamp + timestamp_offset, is_primary,
- std::move(frame));
+ new LegacyEncodedAudioFrame(decoder, std::move(new_payload)));
+ results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame));
}
}
diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h
index 1c9c3f5..466c8b3 100644
--- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h
+++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h
@@ -20,16 +20,13 @@
class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame {
public:
- LegacyEncodedAudioFrame(AudioDecoder* decoder,
- rtc::Buffer&& payload,
- bool is_primary_payload);
+ LegacyEncodedAudioFrame(AudioDecoder* decoder, rtc::Buffer&& payload);
~LegacyEncodedAudioFrame() override;
static std::vector<AudioDecoder::ParseResult> SplitBySamples(
AudioDecoder* decoder,
rtc::Buffer&& payload,
uint32_t timestamp,
- bool is_primary,
size_t bytes_per_ms,
uint32_t timestamps_per_ms);
@@ -44,7 +41,6 @@
private:
AudioDecoder* const decoder_;
const rtc::Buffer payload_;
- const bool is_primary_payload_;
};
} // namespace webrtc
diff --git a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc
index 2a4d9ed..9a07563 100644
--- a/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc
+++ b/webrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc
@@ -123,7 +123,7 @@
const auto results = LegacyEncodedAudioFrame::SplitBySamples(
nullptr,
generate_payload(expected_split.payload_size_ms * bytes_per_ms_),
- kBaseTimestamp, true, bytes_per_ms_, samples_per_ms_);
+ kBaseTimestamp, bytes_per_ms_, samples_per_ms_);
EXPECT_EQ(expected_split.num_frames, results.size());
uint32_t expected_timestamp = kBaseTimestamp;
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc
index 42abd0a..b6d8a3a 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc
@@ -10,10 +10,60 @@
#include "webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h"
+#include <utility>
+
#include "webrtc/base/checks.h"
namespace webrtc {
+namespace {
+class OpusFrame : public AudioDecoder::EncodedAudioFrame {
+ public:
+ OpusFrame(AudioDecoderOpus* decoder,
+ rtc::Buffer&& payload,
+ bool is_primary_payload)
+ : decoder_(decoder),
+ payload_(std::move(payload)),
+ is_primary_payload_(is_primary_payload) {}
+
+ size_t Duration() const override {
+ int ret;
+ if (is_primary_payload_) {
+ ret = decoder_->PacketDuration(payload_.data(), payload_.size());
+ } else {
+ ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size());
+ }
+ return (ret < 0) ? 0 : static_cast<size_t>(ret);
+ }
+
+ rtc::Optional<DecodeResult> Decode(
+ rtc::ArrayView<int16_t> decoded) const override {
+ AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech;
+ int ret;
+ if (is_primary_payload_) {
+ ret = decoder_->Decode(
+ payload_.data(), payload_.size(), decoder_->SampleRateHz(),
+ decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
+ } else {
+ ret = decoder_->DecodeRedundant(
+ payload_.data(), payload_.size(), decoder_->SampleRateHz(),
+ decoded.size() * sizeof(int16_t), decoded.data(), &speech_type);
+ }
+
+ if (ret < 0)
+ return rtc::Optional<DecodeResult>();
+
+ return rtc::Optional<DecodeResult>({static_cast<size_t>(ret), speech_type});
+ }
+
+ private:
+ AudioDecoderOpus* const decoder_;
+ const rtc::Buffer payload_;
+ const bool is_primary_payload_;
+};
+
+} // namespace
+
AudioDecoderOpus::AudioDecoderOpus(size_t num_channels)
: channels_(num_channels) {
RTC_DCHECK(num_channels == 1 || num_channels == 2);
@@ -25,6 +75,26 @@
WebRtcOpus_DecoderFree(dec_state_);
}
+std::vector<AudioDecoder::ParseResult> AudioDecoderOpus::ParsePayload(
+ rtc::Buffer&& payload,
+ uint32_t timestamp) {
+ std::vector<ParseResult> results;
+
+ if (PacketHasFec(payload.data(), payload.size())) {
+ const int duration =
+ PacketDurationRedundant(payload.data(), payload.size());
+ RTC_DCHECK_GE(duration, 0);
+ rtc::Buffer payload_copy(payload.data(), payload.size());
+ std::unique_ptr<EncodedAudioFrame> fec_frame(
+ new OpusFrame(this, std::move(payload_copy), false));
+ results.emplace_back(timestamp - duration, 1, std::move(fec_frame));
+ }
+ std::unique_ptr<EncodedAudioFrame> frame(
+ new OpusFrame(this, std::move(payload), true));
+ results.emplace_back(timestamp, 0, std::move(frame));
+ return results;
+}
+
int AudioDecoderOpus::DecodeInternal(const uint8_t* encoded,
size_t encoded_len,
int sample_rate_hz,
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h
index c222041..a0fb34c 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h
@@ -22,6 +22,8 @@
explicit AudioDecoderOpus(size_t num_channels);
~AudioDecoderOpus() override;
+ std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
+ uint32_t timestamp) override;
void Reset() override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int PacketDurationRedundant(const uint8_t* encoded,
diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc
index e600d2d..43d2dac 100644
--- a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc
+++ b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc
@@ -47,12 +47,11 @@
std::vector<AudioDecoder::ParseResult> AudioDecoderPcm16B::ParsePayload(
rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) {
+ uint32_t timestamp) {
const int samples_per_ms = rtc::CheckedDivExact(sample_rate_hz_, 1000);
return LegacyEncodedAudioFrame::SplitBySamples(
- this, std::move(payload), timestamp, is_primary,
- samples_per_ms * 2 * num_channels_, samples_per_ms);
+ this, std::move(payload), timestamp, samples_per_ms * 2 * num_channels_,
+ samples_per_ms);
}
int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded,
diff --git a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h
index 1d2da4a..577eecc5 100644
--- a/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h
+++ b/webrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h
@@ -21,8 +21,7 @@
AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels);
void Reset() override;
std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
- uint32_t timestamp,
- bool is_primary) override;
+ uint32_t timestamp) override;
int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override;
int SampleRateHz() const override;
size_t Channels() const override;
diff --git a/webrtc/modules/audio_coding/neteq/include/neteq.h b/webrtc/modules/audio_coding/neteq/include/neteq.h
index 98bb37c..c07143c 100644
--- a/webrtc/modules/audio_coding/neteq/include/neteq.h
+++ b/webrtc/modules/audio_coding/neteq/include/neteq.h
@@ -126,7 +126,6 @@
kStereoNotSupported,
kSampleUnderrun,
kDecodedTooMuch,
- kFrameSplitError,
kRedundancySplitError,
kPacketBufferCorruption
};
diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h b/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h
deleted file mode 100644
index cf06134..0000000
--- a/webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_
-#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_
-
-#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
-
-#include "testing/gmock/include/gmock/gmock.h"
-
-namespace webrtc {
-
-class MockPayloadSplitter : public PayloadSplitter {
- public:
- MOCK_METHOD1(SplitRed,
- int(PacketList* packet_list));
- MOCK_METHOD2(SplitFec,
- int(PacketList* packet_list, DecoderDatabase* decoder_database));
- MOCK_METHOD2(CheckRedPayloads,
- int(PacketList* packet_list, const DecoderDatabase& decoder_database));
-};
-
-} // namespace webrtc
-#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PAYLOAD_SPLITTER_H_
diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h b/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h
new file mode 100644
index 0000000..e1f4e04
--- /dev/null
+++ b/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_
+#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_
+
+#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
+
+#include "testing/gmock/include/gmock/gmock.h"
+
+namespace webrtc {
+
+class MockRedPayloadSplitter : public RedPayloadSplitter {
+ public:
+ MOCK_METHOD1(SplitRed, bool(PacketList* packet_list));
+ MOCK_METHOD2(CheckRedPayloads,
+ int(PacketList* packet_list,
+ const DecoderDatabase& decoder_database));
+};
+
+} // namespace webrtc
+#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_
diff --git a/webrtc/modules/audio_coding/neteq/neteq.gypi b/webrtc/modules/audio_coding/neteq/neteq.gypi
index 630076e..90aaaed 100644
--- a/webrtc/modules/audio_coding/neteq/neteq.gypi
+++ b/webrtc/modules/audio_coding/neteq/neteq.gypi
@@ -113,8 +113,8 @@
'packet.h',
'packet_buffer.cc',
'packet_buffer.h',
- 'payload_splitter.cc',
- 'payload_splitter.h',
+ 'red_payload_splitter.cc',
+ 'red_payload_splitter.h',
'post_decode_vad.cc',
'post_decode_vad.h',
'preemptive_expand.cc',
diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc
index 221b07c..6b2fa4c 100644
--- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc
+++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc
@@ -41,7 +41,7 @@
#include "webrtc/modules/audio_coding/neteq/normal.h"
#include "webrtc/modules/audio_coding/neteq/packet_buffer.h"
#include "webrtc/modules/audio_coding/neteq/packet.h"
-#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
+#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
#include "webrtc/modules/audio_coding/neteq/post_decode_vad.h"
#include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
@@ -65,7 +65,7 @@
dtmf_tone_generator(new DtmfToneGenerator),
packet_buffer(
new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())),
- payload_splitter(new PayloadSplitter),
+ red_payload_splitter(new RedPayloadSplitter),
timestamp_scaler(new TimestampScaler(*decoder_database)),
accelerate_factory(new AccelerateFactory),
expand_factory(new ExpandFactory),
@@ -84,7 +84,7 @@
dtmf_buffer_(std::move(deps.dtmf_buffer)),
dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)),
packet_buffer_(std::move(deps.packet_buffer)),
- payload_splitter_(std::move(deps.payload_splitter)),
+ red_payload_splitter_(std::move(deps.red_payload_splitter)),
timestamp_scaler_(std::move(deps.timestamp_scaler)),
vad_(new PostDecodeVad()),
expand_factory_(std::move(deps.expand_factory)),
@@ -567,7 +567,6 @@
packet->header.ssrc = rtp_header.header.ssrc;
packet->header.numCSRCs = 0;
packet->payload.SetData(payload.data(), payload.size());
- packet->primary = true;
// Waiting time will be set upon inserting the packet in the buffer.
RTC_DCHECK(!packet->waiting_time);
// Insert packet in a packet list.
@@ -609,13 +608,13 @@
// Check for RED payload type, and separate payloads into several packets.
if (decoder_database_->IsRed(main_header.payloadType)) {
- if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) {
+ if (!red_payload_splitter_->SplitRed(&packet_list)) {
PacketBuffer::DeleteAllPackets(&packet_list);
return kRedundancySplitError;
}
// Only accept a few RED payloads of the same type as the main data,
// DTMF events and CNG.
- payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
+ red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
// Update the stored main payload header since the main payload has now
// changed.
memcpy(&main_header, &packet_list.front()->header, sizeof(main_header));
@@ -658,18 +657,6 @@
}
}
- // Check for FEC in packets, and separate payloads into several packets.
- int ret = payload_splitter_->SplitFec(&packet_list, decoder_database_.get());
- if (ret != PayloadSplitter::kOK) {
- PacketBuffer::DeleteAllPackets(&packet_list);
- switch (ret) {
- case PayloadSplitter::kUnknownPayloadType:
- return kUnknownRtpPayloadType;
- default:
- return kOtherError;
- }
- }
-
// Update bandwidth estimate, if the packet is not comfort noise.
if (!packet_list.empty() &&
!decoder_database_->IsComfortNoise(main_header.payloadType)) {
@@ -702,9 +689,9 @@
} else {
std::vector<AudioDecoder::ParseResult> results =
info->GetDecoder()->ParsePayload(std::move(packet->payload),
- packet->header.timestamp,
- packet->primary);
+ packet->header.timestamp);
const RTPHeader& original_header = packet->header;
+ const Packet::Priority original_priority = packet->priority;
for (auto& result : results) {
RTC_DCHECK(result.frame);
// Reuse the packet if possible.
@@ -713,8 +700,9 @@
packet->header = original_header;
}
packet->header.timestamp = result.timestamp;
- // TODO(ossu): Move from primary to some sort of priority level.
- packet->primary = result.primary;
+ RTC_DCHECK_GE(result.priority, 0);
+ packet->priority.codec_level = result.priority;
+ packet->priority.red_level = original_priority.red_level;
packet->frame = std::move(result.frame);
parsed_packet_list.push_back(packet.release());
}
@@ -734,7 +722,7 @@
// Insert packets in buffer.
const size_t buffer_length_before_insert =
packet_buffer_->NumPacketsInBuffer();
- ret = packet_buffer_->InsertPacketList(
+ const int ret = packet_buffer_->InsertPacketList(
&parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_,
¤t_cng_rtp_payload_type_);
if (ret == PacketBuffer::kFlushed) {
@@ -1979,9 +1967,8 @@
size_t packet_duration = 0;
if (packet->frame) {
packet_duration = packet->frame->Duration();
- // TODO(ossu): Is this the correct way to track samples decoded from a
- // redundant packet?
- if (packet_duration > 0 && !packet->primary) {
+ // TODO(ossu): Is this the correct way to track Opus FEC packets?
+ if (packet->priority.codec_level > 0) {
stats_.SecondaryDecodedSamples(rtc::checked_cast<int>(packet_duration));
}
} else if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) {
diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.h b/webrtc/modules/audio_coding/neteq/neteq_impl.h
index dd35301..dda3d64 100644
--- a/webrtc/modules/audio_coding/neteq/neteq_impl.h
+++ b/webrtc/modules/audio_coding/neteq/neteq_impl.h
@@ -46,7 +46,7 @@
class NackTracker;
class Normal;
class PacketBuffer;
-class PayloadSplitter;
+class RedPayloadSplitter;
class PostDecodeVad;
class PreemptiveExpand;
class RandomVector;
@@ -86,7 +86,7 @@
std::unique_ptr<DtmfBuffer> dtmf_buffer;
std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator;
std::unique_ptr<PacketBuffer> packet_buffer;
- std::unique_ptr<PayloadSplitter> payload_splitter;
+ std::unique_ptr<RedPayloadSplitter> red_payload_splitter;
std::unique_ptr<TimestampScaler> timestamp_scaler;
std::unique_ptr<AccelerateFactory> accelerate_factory;
std::unique_ptr<ExpandFactory> expand_factory;
@@ -353,7 +353,7 @@
const std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator_
GUARDED_BY(crit_sect_);
const std::unique_ptr<PacketBuffer> packet_buffer_ GUARDED_BY(crit_sect_);
- const std::unique_ptr<PayloadSplitter> payload_splitter_
+ const std::unique_ptr<RedPayloadSplitter> red_payload_splitter_
GUARDED_BY(crit_sect_);
const std::unique_ptr<TimestampScaler> timestamp_scaler_
GUARDED_BY(crit_sect_);
diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc
index 5b1015d..e3029db 100644
--- a/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc
@@ -28,7 +28,7 @@
#include "webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h"
#include "webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h"
#include "webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h"
-#include "webrtc/modules/audio_coding/neteq/mock/mock_payload_splitter.h"
+#include "webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h"
#include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
#include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h"
@@ -124,11 +124,11 @@
packet_buffer_ = deps.packet_buffer.get();
if (use_mock_payload_splitter_) {
- std::unique_ptr<MockPayloadSplitter> mock(new MockPayloadSplitter);
+ std::unique_ptr<MockRedPayloadSplitter> mock(new MockRedPayloadSplitter);
mock_payload_splitter_ = mock.get();
- deps.payload_splitter = std::move(mock);
+ deps.red_payload_splitter = std::move(mock);
}
- payload_splitter_ = deps.payload_splitter.get();
+ red_payload_splitter_ = deps.red_payload_splitter.get();
deps.timestamp_scaler = std::unique_ptr<TimestampScaler>(
new TimestampScaler(*deps.decoder_database.get()));
@@ -197,8 +197,8 @@
MockPacketBuffer* mock_packet_buffer_ = nullptr;
PacketBuffer* packet_buffer_ = nullptr;
bool use_mock_packet_buffer_ = true;
- MockPayloadSplitter* mock_payload_splitter_ = nullptr;
- PayloadSplitter* payload_splitter_ = nullptr;
+ MockRedPayloadSplitter* mock_payload_splitter_ = nullptr;
+ RedPayloadSplitter* red_payload_splitter_ = nullptr;
bool use_mock_payload_splitter_ = true;
};
@@ -332,11 +332,6 @@
.WillOnce(Return(0));
}
- // Expectations for payload splitter.
- EXPECT_CALL(*mock_payload_splitter_, SplitFec(_, _))
- .Times(2)
- .WillRepeatedly(Return(PayloadSplitter::kOK));
-
// Insert first packet.
neteq_->InsertPacket(rtp_header, payload, kFirstReceiveTime);
diff --git a/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc b/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc
index 15c60a8..70424fd 100644
--- a/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc
@@ -39,18 +39,57 @@
MOCK_METHOD0(Reset, void());
- int PacketDuration(const uint8_t* encoded,
- size_t encoded_len) const /* override */ {
- return kPacketDuration;
+ class MockFrame : public AudioDecoder::EncodedAudioFrame {
+ public:
+ MockFrame(size_t num_channels) : num_channels_(num_channels) {}
+
+ size_t Duration() const override { return kPacketDuration; }
+
+ rtc::Optional<DecodeResult> Decode(
+ rtc::ArrayView<int16_t> decoded) const override {
+ const size_t output_size =
+ sizeof(int16_t) * kPacketDuration * num_channels_;
+ if (decoded.size() >= output_size) {
+ memset(decoded.data(), 0,
+ sizeof(int16_t) * kPacketDuration * num_channels_);
+ return rtc::Optional<DecodeResult>(
+ {kPacketDuration * num_channels_, kSpeech});
+ } else {
+ ADD_FAILURE() << "Expected decoded.size() to be >= output_size ("
+ << decoded.size() << " vs. " << output_size << ")";
+ return rtc::Optional<DecodeResult>();
+ }
+ }
+
+ private:
+ const size_t num_channels_;
+ };
+
+ std::vector<ParseResult> ParsePayload(rtc::Buffer&& payload,
+ uint32_t timestamp) /* override */ {
+ std::vector<ParseResult> results;
+ if (fec_enabled_) {
+ std::unique_ptr<MockFrame> fec_frame(new MockFrame(num_channels_));
+ results.emplace_back(timestamp - kPacketDuration, 1,
+ std::move(fec_frame));
+ }
+
+ std::unique_ptr<MockFrame> frame(new MockFrame(num_channels_));
+ results.emplace_back(timestamp, 0, std::move(frame));
+ return results;
}
- int PacketDurationRedundant(const uint8_t* encoded,
- size_t encoded_len) const /* override */ {
+ int PacketDuration(const uint8_t* encoded, size_t encoded_len) const
+ /* override */ {
+ ADD_FAILURE() << "Since going through ParsePayload, PacketDuration should "
+ "never get called.";
return kPacketDuration;
}
bool PacketHasFec(
const uint8_t* encoded, size_t encoded_len) const /* override */ {
+ ADD_FAILURE() << "Since going through ParsePayload, PacketHasFec should "
+ "never get called.";
return fec_enabled_;
}
@@ -63,24 +102,14 @@
bool fec_enabled() const { return fec_enabled_; }
protected:
- // Override the following methods such that no actual payload is needed.
int DecodeInternal(const uint8_t* encoded,
size_t encoded_len,
- int /*sample_rate_hz*/,
+ int sample_rate_hz,
int16_t* decoded,
SpeechType* speech_type) /* override */ {
- *speech_type = kSpeech;
- memset(decoded, 0, sizeof(int16_t) * kPacketDuration * Channels());
- return kPacketDuration * Channels();
- }
-
- int DecodeRedundantInternal(const uint8_t* encoded,
- size_t encoded_len,
- int sample_rate_hz,
- int16_t* decoded,
- SpeechType* speech_type) /* override */ {
- return DecodeInternal(encoded, encoded_len, sample_rate_hz, decoded,
- speech_type);
+ ADD_FAILURE() << "Since going through ParsePayload, DecodeInternal should "
+ "never get called.";
+ return -1;
}
private:
diff --git a/webrtc/modules/audio_coding/neteq/packet.h b/webrtc/modules/audio_coding/neteq/packet.h
index 4e17a88..cf590ca 100644
--- a/webrtc/modules/audio_coding/neteq/packet.h
+++ b/webrtc/modules/audio_coding/neteq/packet.h
@@ -24,10 +24,52 @@
// Struct for holding RTP packets.
struct Packet {
+ struct Priority {
+ Priority() : codec_level(0), red_level(0) {}
+ Priority(int codec_level, int red_level)
+ : codec_level(codec_level), red_level(red_level) {
+ CheckInvariant();
+ }
+
+ int codec_level;
+ int red_level;
+
+ // Priorities are sorted low-to-high, first on the level the codec
+ // prioritizes it, then on the level of RED packet it is; i.e. if it is a
+ // primary or secondary payload of a RED packet. For example: with Opus, an
+ // Fec packet (which the decoder prioritizes lower than a regular packet)
+ // will not be used if there is _any_ RED payload for the same
+ // timeframe. The highest priority packet will have levels {0, 0}. Negative
+ // priorities are not allowed.
+ bool operator<(const Priority& b) const {
+ CheckInvariant();
+ b.CheckInvariant();
+ if (codec_level == b.codec_level)
+ return red_level < b.red_level;
+
+ return codec_level < b.codec_level;
+ }
+ bool operator==(const Priority& b) const {
+ CheckInvariant();
+ b.CheckInvariant();
+ return codec_level == b.codec_level && red_level == b.red_level;
+ }
+ bool operator!=(const Priority& b) const { return !(*this == b); }
+ bool operator>(const Priority& b) const { return b < *this; }
+ bool operator<=(const Priority& b) const { return !(b > *this); }
+ bool operator>=(const Priority& b) const { return !(b < *this); }
+
+ private:
+ void CheckInvariant() const {
+ RTC_DCHECK_GE(codec_level, 0);
+ RTC_DCHECK_GE(red_level, 0);
+ }
+ };
+
RTPHeader header;
// Datagram excluding RTP header and header extension.
rtc::Buffer payload;
- bool primary = true; // Primary, i.e., not redundant payload.
+ Priority priority;
std::unique_ptr<TickTimer::Stopwatch> waiting_time;
std::unique_ptr<AudioDecoder::EncodedAudioFrame> frame;
@@ -41,17 +83,16 @@
// primary payload is considered "smaller" than a secondary.
bool operator==(const Packet& rhs) const {
return (this->header.timestamp == rhs.header.timestamp &&
- this->header.sequenceNumber == rhs.header.sequenceNumber &&
- this->primary == rhs.primary);
+ this->header.sequenceNumber == rhs.header.sequenceNumber &&
+ this->priority == rhs.priority);
}
bool operator!=(const Packet& rhs) const { return !operator==(rhs); }
bool operator<(const Packet& rhs) const {
if (this->header.timestamp == rhs.header.timestamp) {
if (this->header.sequenceNumber == rhs.header.sequenceNumber) {
- // Timestamp and sequence numbers are identical - deem the left
- // hand side to be "smaller" (i.e., "earlier") if it is primary, and
- // right hand side is not.
- return (this->primary && !rhs.primary);
+ // Timestamp and sequence numbers are identical - deem the left hand
+ // side to be "smaller" (i.e., "earlier") if it has higher priority.
+ return this->priority < rhs.priority;
}
return (static_cast<uint16_t>(rhs.header.sequenceNumber
- this->header.sequenceNumber) < 0xFFFF / 2);
diff --git a/webrtc/modules/audio_coding/neteq/packet_buffer.cc b/webrtc/modules/audio_coding/neteq/packet_buffer.cc
index c5b23dc..eeb1d27 100644
--- a/webrtc/modules/audio_coding/neteq/packet_buffer.cc
+++ b/webrtc/modules/audio_coding/neteq/packet_buffer.cc
@@ -76,6 +76,9 @@
return kInvalidPacket;
}
+ RTC_DCHECK_GE(packet->priority.codec_level, 0);
+ RTC_DCHECK_GE(packet->priority.red_level, 0);
+
int return_val = kOK;
packet->waiting_time = tick_timer_->GetNewStopwatch();
@@ -262,7 +265,7 @@
void PacketBuffer::DiscardPacketsWithPayloadType(uint8_t payload_type) {
for (auto it = buffer_.begin(); it != buffer_.end(); /* */) {
- Packet *packet = *it;
+ Packet* packet = *it;
if (packet->header.payloadType == payload_type) {
delete packet;
it = buffer_.erase(it);
@@ -281,7 +284,9 @@
size_t last_duration = last_decoded_length;
for (Packet* packet : buffer_) {
if (packet->frame) {
- if (!packet->primary) {
+ // TODO(hlundin): Verify that it's fine to count all packets and remove
+ // this check.
+ if (packet->priority != Packet::Priority(0, 0)) {
continue;
}
size_t duration = packet->frame->Duration();
diff --git a/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc b/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc
index 1b86d83..b39169f 100644
--- a/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc
@@ -60,7 +60,6 @@
packet->header.ssrc = 0x12345678;
packet->header.numCSRCs = 0;
packet->header.paddingLength = 0;
- packet->primary = true;
packet->payload.SetSize(payload_size_bytes);
++seq_no_;
ts_ += frame_size_;
@@ -284,7 +283,7 @@
packet_facts[i].payload_type,
kFrameSize);
Packet* packet = gen.NextPacket(kPayloadLength);
- packet->primary = packet_facts[i].primary;
+ packet->priority.red_level = packet_facts[i].primary ? 0 : 1;
EXPECT_EQ(PacketBuffer::kOK, buffer.InsertPacket(packet));
if (packet_facts[i].extract_order >= 0) {
expect_order[packet_facts[i].extract_order] = packet;
@@ -512,8 +511,8 @@
// The function should return true if the first packet "goes before" the second.
TEST(PacketBuffer, ComparePackets) {
PacketGenerator gen(0, 0, 0, 10);
- Packet* a = gen.NextPacket(10); // SN = 0, TS = 0.
- Packet* b = gen.NextPacket(10); // SN = 1, TS = 10.
+ std::unique_ptr<Packet> a(gen.NextPacket(10)); // SN = 0, TS = 0.
+ std::unique_ptr<Packet> b(gen.NextPacket(10)); // SN = 1, TS = 10.
EXPECT_FALSE(*a == *b);
EXPECT_TRUE(*a != *b);
EXPECT_TRUE(*a < *b);
@@ -556,10 +555,11 @@
EXPECT_TRUE(*a <= *b);
EXPECT_FALSE(*a >= *b);
- // Test equal timestamps and sequence numbers, but only 'b' is primary.
+ // Test equal timestamps and sequence numbers, but differing priorities.
a->header.sequenceNumber = b->header.sequenceNumber;
- a->primary = false;
- b->primary = true;
+ a->priority = {1, 0};
+ b->priority = {0, 0};
+ // a after b
EXPECT_FALSE(*a == *b);
EXPECT_TRUE(*a != *b);
EXPECT_FALSE(*a < *b);
@@ -567,8 +567,53 @@
EXPECT_FALSE(*a <= *b);
EXPECT_TRUE(*a >= *b);
- delete a;
- delete b;
+ std::unique_ptr<Packet> c(gen.NextPacket(0)); // SN = 2, TS = 20.
+ std::unique_ptr<Packet> d(gen.NextPacket(0)); // SN = 3, TS = 20.
+ c->header.timestamp = b->header.timestamp;
+ d->header.timestamp = b->header.timestamp;
+ c->header.sequenceNumber = b->header.sequenceNumber;
+ d->header.sequenceNumber = b->header.sequenceNumber;
+ c->priority = {1, 1};
+ d->priority = {0, 1};
+ // c after d
+ EXPECT_FALSE(*c == *d);
+ EXPECT_TRUE(*c != *d);
+ EXPECT_FALSE(*c < *d);
+ EXPECT_TRUE(*c > *d);
+ EXPECT_FALSE(*c <= *d);
+ EXPECT_TRUE(*c >= *d);
+
+ // c after a
+ EXPECT_FALSE(*c == *a);
+ EXPECT_TRUE(*c != *a);
+ EXPECT_FALSE(*c < *a);
+ EXPECT_TRUE(*c > *a);
+ EXPECT_FALSE(*c <= *a);
+ EXPECT_TRUE(*c >= *a);
+
+ // c after b
+ EXPECT_FALSE(*c == *b);
+ EXPECT_TRUE(*c != *b);
+ EXPECT_FALSE(*c < *b);
+ EXPECT_TRUE(*c > *b);
+ EXPECT_FALSE(*c <= *b);
+ EXPECT_TRUE(*c >= *b);
+
+ // a after d
+ EXPECT_FALSE(*a == *d);
+ EXPECT_TRUE(*a != *d);
+ EXPECT_FALSE(*a < *d);
+ EXPECT_TRUE(*a > *d);
+ EXPECT_FALSE(*a <= *d);
+ EXPECT_TRUE(*a >= *d);
+
+ // d after b
+ EXPECT_FALSE(*d == *b);
+ EXPECT_TRUE(*d != *b);
+ EXPECT_FALSE(*d < *b);
+ EXPECT_TRUE(*d > *b);
+ EXPECT_FALSE(*d <= *b);
+ EXPECT_TRUE(*d >= *b);
}
// Test the DeleteFirstPacket DeleteAllPackets methods.
diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.cc b/webrtc/modules/audio_coding/neteq/payload_splitter.cc
deleted file mode 100644
index 28e561a..0000000
--- a/webrtc/modules/audio_coding/neteq/payload_splitter.cc
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
-
-#include <assert.h>
-
-#include "webrtc/base/checks.h"
-#include "webrtc/base/logging.h"
-#include "webrtc/modules/audio_coding/neteq/decoder_database.h"
-
-namespace webrtc {
-
-// The method loops through a list of packets {A, B, C, ...}. Each packet is
-// split into its corresponding RED payloads, {A1, A2, ...}, which is
-// temporarily held in the list |new_packets|.
-// When the first packet in |packet_list| has been processed, the orignal packet
-// is replaced by the new ones in |new_packets|, so that |packet_list| becomes:
-// {A1, A2, ..., B, C, ...}. The method then continues with B, and C, until all
-// the original packets have been replaced by their split payloads.
-int PayloadSplitter::SplitRed(PacketList* packet_list) {
- int ret = kOK;
- PacketList::iterator it = packet_list->begin();
- while (it != packet_list->end()) {
- const Packet* red_packet = (*it);
- assert(!red_packet->payload.empty());
- const uint8_t* payload_ptr = red_packet->payload.data();
-
- // Read RED headers (according to RFC 2198):
- //
- // 0 1 2 3
- // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
- // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- // |F| block PT | timestamp offset | block length |
- // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- // Last RED header:
- // 0 1 2 3 4 5 6 7
- // +-+-+-+-+-+-+-+-+
- // |0| Block PT |
- // +-+-+-+-+-+-+-+-+
-
- struct RedHeader {
- uint8_t payload_type;
- uint32_t timestamp;
- size_t payload_length;
- bool primary;
- };
-
- std::vector<RedHeader> new_headers;
- bool last_block = false;
- size_t sum_length = 0;
- while (!last_block) {
- RedHeader new_header;
- // Check the F bit. If F == 0, this was the last block.
- last_block = ((*payload_ptr & 0x80) == 0);
- // Bits 1 through 7 are payload type.
- new_header.payload_type = payload_ptr[0] & 0x7F;
- if (last_block) {
- // No more header data to read.
- ++sum_length; // Account for RED header size of 1 byte.
- new_header.timestamp = red_packet->header.timestamp;
- new_header.payload_length = red_packet->payload.size() - sum_length;
- new_header.primary = true; // Last block is always primary.
- payload_ptr += 1; // Advance to first payload byte.
- } else {
- // Bits 8 through 21 are timestamp offset.
- int timestamp_offset = (payload_ptr[1] << 6) +
- ((payload_ptr[2] & 0xFC) >> 2);
- new_header.timestamp = red_packet->header.timestamp - timestamp_offset;
- // Bits 22 through 31 are payload length.
- new_header.payload_length =
- ((payload_ptr[2] & 0x03) << 8) + payload_ptr[3];
- new_header.primary = false;
- payload_ptr += 4; // Advance to next RED header.
- }
- sum_length += new_header.payload_length;
- sum_length += 4; // Account for RED header size of 4 bytes.
- // Store in new list of packets.
- new_headers.push_back(new_header);
- }
-
- // Populate the new packets with payload data.
- // |payload_ptr| now points at the first payload byte.
- PacketList new_packets; // An empty list to store the split packets in.
- for (const auto& new_header : new_headers) {
- size_t payload_length = new_header.payload_length;
- if (payload_ptr + payload_length >
- red_packet->payload.data() + red_packet->payload.size()) {
- // The block lengths in the RED headers do not match the overall packet
- // length. Something is corrupt. Discard this and the remaining
- // payloads from this packet.
- LOG(LS_WARNING) << "SplitRed length mismatch";
- ret = kRedLengthMismatch;
- break;
- }
- Packet* new_packet = new Packet;
- new_packet->header = red_packet->header;
- new_packet->header.timestamp = new_header.timestamp;
- new_packet->header.payloadType = new_header.payload_type;
- new_packet->primary = new_header.primary;
- new_packet->payload.SetData(payload_ptr, payload_length);
- new_packets.push_front(new_packet);
- payload_ptr += payload_length;
- }
- // Insert new packets into original list, before the element pointed to by
- // iterator |it|.
- packet_list->splice(it, new_packets, new_packets.begin(),
- new_packets.end());
- // Delete old packet payload.
- delete (*it);
- // Remove |it| from the packet list. This operation effectively moves the
- // iterator |it| to the next packet in the list. Thus, we do not have to
- // increment it manually.
- it = packet_list->erase(it);
- }
- return ret;
-}
-
-int PayloadSplitter::SplitFec(PacketList* packet_list,
- DecoderDatabase* decoder_database) {
- PacketList::iterator it = packet_list->begin();
- // Iterate through all packets in |packet_list|.
- while (it != packet_list->end()) {
- Packet* packet = (*it); // Just to make the notation more intuitive.
- // Get codec type for this payload.
- uint8_t payload_type = packet->header.payloadType;
- const DecoderDatabase::DecoderInfo* info =
- decoder_database->GetDecoderInfo(payload_type);
- if (!info) {
- LOG(LS_WARNING) << "SplitFec unknown payload type";
- return kUnknownPayloadType;
- }
-
- // Not an FEC packet.
- AudioDecoder* decoder = decoder_database->GetDecoder(payload_type);
- // decoder should not return NULL, except for comfort noise payloads which
- // are handled separately.
- assert(decoder != NULL || decoder_database->IsComfortNoise(payload_type));
- if (!decoder ||
- !decoder->PacketHasFec(packet->payload.data(),
- packet->payload.size())) {
- ++it;
- continue;
- }
-
- switch (info->codec_type) {
- case NetEqDecoder::kDecoderOpus:
- case NetEqDecoder::kDecoderOpus_2ch: {
- // The main payload of this packet should be decoded as a primary
- // payload, even if it comes as a secondary payload in a RED packet.
- packet->primary = true;
-
- Packet* new_packet = new Packet;
- new_packet->header = packet->header;
- int duration = decoder->PacketDurationRedundant(packet->payload.data(),
- packet->payload.size());
- new_packet->header.timestamp -= duration;
- new_packet->payload.SetData(packet->payload);
- new_packet->primary = false;
- // Waiting time should not be set here.
- RTC_DCHECK(!packet->waiting_time);
-
- packet_list->insert(it, new_packet);
- break;
- }
- default: {
- LOG(LS_WARNING) << "SplitFec wrong payload type";
- return kFecSplitError;
- }
- }
-
- ++it;
- }
- return kOK;
-}
-
-int PayloadSplitter::CheckRedPayloads(PacketList* packet_list,
- const DecoderDatabase& decoder_database) {
- PacketList::iterator it = packet_list->begin();
- int main_payload_type = -1;
- int num_deleted_packets = 0;
- while (it != packet_list->end()) {
- uint8_t this_payload_type = (*it)->header.payloadType;
- if (!decoder_database.IsDtmf(this_payload_type) &&
- !decoder_database.IsComfortNoise(this_payload_type)) {
- if (main_payload_type == -1) {
- // This is the first packet in the list which is non-DTMF non-CNG.
- main_payload_type = this_payload_type;
- } else {
- if (this_payload_type != main_payload_type) {
- // We do not allow redundant payloads of a different type.
- // Discard this payload.
- delete (*it);
- // Remove |it| from the packet list. This operation effectively
- // moves the iterator |it| to the next packet in the list. Thus, we
- // do not have to increment it manually.
- it = packet_list->erase(it);
- ++num_deleted_packets;
- continue;
- }
- }
- }
- ++it;
- }
- return num_deleted_packets;
-}
-
-} // namespace webrtc
diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.h b/webrtc/modules/audio_coding/neteq/payload_splitter.h
deleted file mode 100644
index a3e1b1d..0000000
--- a/webrtc/modules/audio_coding/neteq/payload_splitter.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_
-#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_
-
-#include "webrtc/base/constructormagic.h"
-#include "webrtc/modules/audio_coding/neteq/packet.h"
-
-namespace webrtc {
-
-// Forward declarations.
-class DecoderDatabase;
-
-// This class handles splitting of payloads into smaller parts.
-
-// For RED and FEC the splitting is done internally. Other codecs' packets are
-// split by calling AudioDecoder::SplitPacket.
-class PayloadSplitter {
- public:
- enum SplitterReturnCodes {
- kOK = 0,
- kNoSplit = 1,
- kFrameSplitError = -2,
- kUnknownPayloadType = -3,
- kRedLengthMismatch = -4,
- kFecSplitError = -5,
- };
-
- PayloadSplitter() {}
-
- virtual ~PayloadSplitter() {}
-
- // Splits each packet in |packet_list| into its separate RED payloads. Each
- // RED payload is packetized into a Packet. The original elements in
- // |packet_list| are properly deleted, and replaced by the new packets.
- // Note that all packets in |packet_list| must be RED payloads, i.e., have
- // RED headers according to RFC 2198 at the very beginning of the payload.
- // Returns kOK or an error.
- virtual int SplitRed(PacketList* packet_list);
-
- // Iterates through |packet_list| and, duplicate each audio payload that has
- // FEC as new packet for redundant decoding. The decoder database is needed to
- // get information about which payload type each packet contains.
- virtual int SplitFec(PacketList* packet_list,
- DecoderDatabase* decoder_database);
-
- // Checks all packets in |packet_list|. Packets that are DTMF events or
- // comfort noise payloads are kept. Except that, only one single payload type
- // is accepted. Any packet with another payload type is discarded.
- virtual int CheckRedPayloads(PacketList* packet_list,
- const DecoderDatabase& decoder_database);
-
- private:
- RTC_DISALLOW_COPY_AND_ASSIGN(PayloadSplitter);
-};
-
-} // namespace webrtc
-#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_PAYLOAD_SPLITTER_H_
diff --git a/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc b/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc
new file mode 100644
index 0000000..a2f1098
--- /dev/null
+++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter.cc
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
+
+#include <assert.h>
+#include <vector>
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/base/safe_conversions.h"
+#include "webrtc/modules/audio_coding/neteq/decoder_database.h"
+
+namespace webrtc {
+
+// The method loops through a list of packets {A, B, C, ...}. Each packet is
+// split into its corresponding RED payloads, {A1, A2, ...}, which is
+// temporarily held in the list |new_packets|.
+// When the first packet in |packet_list| has been processed, the orignal packet
+// is replaced by the new ones in |new_packets|, so that |packet_list| becomes:
+// {A1, A2, ..., B, C, ...}. The method then continues with B, and C, until all
+// the original packets have been replaced by their split payloads.
+bool RedPayloadSplitter::SplitRed(PacketList* packet_list) {
+ // Too many RED blocks indicates that something is wrong. Clamp it at some
+ // reasonable value.
+ const size_t kMaxRedBlocks = 32;
+ bool ret = true;
+ PacketList::iterator it = packet_list->begin();
+ while (it != packet_list->end()) {
+ const Packet* red_packet = (*it);
+ assert(!red_packet->payload.empty());
+ const uint8_t* payload_ptr = red_packet->payload.data();
+
+ // Read RED headers (according to RFC 2198):
+ //
+ // 0 1 2 3
+ // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ // |F| block PT | timestamp offset | block length |
+ // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ // Last RED header:
+ // 0 1 2 3 4 5 6 7
+ // +-+-+-+-+-+-+-+-+
+ // |0| Block PT |
+ // +-+-+-+-+-+-+-+-+
+
+ struct RedHeader {
+ uint8_t payload_type;
+ uint32_t timestamp;
+ size_t payload_length;
+ };
+
+ std::vector<RedHeader> new_headers;
+ bool last_block = false;
+ size_t sum_length = 0;
+ while (!last_block) {
+ RedHeader new_header;
+ // Check the F bit. If F == 0, this was the last block.
+ last_block = ((*payload_ptr & 0x80) == 0);
+ // Bits 1 through 7 are payload type.
+ new_header.payload_type = payload_ptr[0] & 0x7F;
+ if (last_block) {
+ // No more header data to read.
+ ++sum_length; // Account for RED header size of 1 byte.
+ new_header.timestamp = red_packet->header.timestamp;
+ new_header.payload_length = red_packet->payload.size() - sum_length;
+ payload_ptr += 1; // Advance to first payload byte.
+ } else {
+ // Bits 8 through 21 are timestamp offset.
+ int timestamp_offset =
+ (payload_ptr[1] << 6) + ((payload_ptr[2] & 0xFC) >> 2);
+ new_header.timestamp = red_packet->header.timestamp - timestamp_offset;
+ // Bits 22 through 31 are payload length.
+ new_header.payload_length =
+ ((payload_ptr[2] & 0x03) << 8) + payload_ptr[3];
+ payload_ptr += 4; // Advance to next RED header.
+ }
+ sum_length += new_header.payload_length;
+ sum_length += 4; // Account for RED header size of 4 bytes.
+ // Store in new list of packets.
+ new_headers.push_back(new_header);
+ }
+
+ if (new_headers.size() <= kMaxRedBlocks) {
+ // Populate the new packets with payload data.
+ // |payload_ptr| now points at the first payload byte.
+ PacketList new_packets; // An empty list to store the split packets in.
+ for (size_t i = 0; i != new_headers.size(); ++i) {
+ const auto& new_header = new_headers[i];
+ size_t payload_length = new_header.payload_length;
+ if (payload_ptr + payload_length >
+ red_packet->payload.data() + red_packet->payload.size()) {
+ // The block lengths in the RED headers do not match the overall
+ // packet length. Something is corrupt. Discard this and the remaining
+ // payloads from this packet.
+ LOG(LS_WARNING) << "SplitRed length mismatch";
+ ret = false;
+ break;
+ }
+
+ Packet* new_packet = new Packet;
+ new_packet->header = red_packet->header;
+ new_packet->header.timestamp = new_header.timestamp;
+ new_packet->header.payloadType = new_header.payload_type;
+ new_packet->priority.red_level =
+ rtc::checked_cast<int>((new_headers.size() - 1) - i);
+ new_packet->payload.SetData(payload_ptr, payload_length);
+ new_packets.push_front(new_packet);
+ payload_ptr += payload_length;
+ }
+ // Insert new packets into original list, before the element pointed to by
+ // iterator |it|.
+ packet_list->splice(it, new_packets, new_packets.begin(),
+ new_packets.end());
+ } else {
+ LOG(LS_WARNING) << "SplitRed too many blocks: " << new_headers.size();
+ ret = false;
+ }
+ // Delete old packet payload.
+ delete (*it);
+ // Remove |it| from the packet list. This operation effectively moves the
+ // iterator |it| to the next packet in the list. Thus, we do not have to
+ // increment it manually.
+ it = packet_list->erase(it);
+ }
+ return ret;
+}
+
+int RedPayloadSplitter::CheckRedPayloads(
+ PacketList* packet_list,
+ const DecoderDatabase& decoder_database) {
+ PacketList::iterator it = packet_list->begin();
+ int main_payload_type = -1;
+ int num_deleted_packets = 0;
+ while (it != packet_list->end()) {
+ uint8_t this_payload_type = (*it)->header.payloadType;
+ if (!decoder_database.IsDtmf(this_payload_type) &&
+ !decoder_database.IsComfortNoise(this_payload_type)) {
+ if (main_payload_type == -1) {
+ // This is the first packet in the list which is non-DTMF non-CNG.
+ main_payload_type = this_payload_type;
+ } else {
+ if (this_payload_type != main_payload_type) {
+ // We do not allow redundant payloads of a different type.
+ // Discard this payload.
+ delete (*it);
+ // Remove |it| from the packet list. This operation effectively
+ // moves the iterator |it| to the next packet in the list. Thus, we
+ // do not have to increment it manually.
+ it = packet_list->erase(it);
+ ++num_deleted_packets;
+ continue;
+ }
+ }
+ }
+ ++it;
+ }
+ return num_deleted_packets;
+}
+
+} // namespace webrtc
diff --git a/webrtc/modules/audio_coding/neteq/red_payload_splitter.h b/webrtc/modules/audio_coding/neteq/red_payload_splitter.h
new file mode 100644
index 0000000..deb86eb
--- /dev/null
+++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_
+#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_
+
+#include "webrtc/base/constructormagic.h"
+#include "webrtc/modules/audio_coding/neteq/packet.h"
+
+namespace webrtc {
+
+// Forward declarations.
+class DecoderDatabase;
+
+// This class handles splitting of RED payloads into smaller parts.
+// Codec-specific packet splitting can be performed by
+// AudioDecoder::ParsePayload.
+class RedPayloadSplitter {
+ public:
+ RedPayloadSplitter() {}
+
+ virtual ~RedPayloadSplitter() {}
+
+ // Splits each packet in |packet_list| into its separate RED payloads. Each
+ // RED payload is packetized into a Packet. The original elements in
+ // |packet_list| are properly deleted, and replaced by the new packets.
+ // Note that all packets in |packet_list| must be RED payloads, i.e., have
+ // RED headers according to RFC 2198 at the very beginning of the payload.
+ // Returns kOK or an error.
+ virtual bool SplitRed(PacketList* packet_list);
+
+ // Checks all packets in |packet_list|. Packets that are DTMF events or
+ // comfort noise payloads are kept. Except that, only one single payload type
+ // is accepted. Any packet with another payload type is discarded. Returns
+ // the number of discarded packets.
+ virtual int CheckRedPayloads(PacketList* packet_list,
+ const DecoderDatabase& decoder_database);
+
+ private:
+ RTC_DISALLOW_COPY_AND_ASSIGN(RedPayloadSplitter);
+};
+
+} // namespace webrtc
+#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_
diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc b/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc
similarity index 70%
rename from webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc
rename to webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc
index 0a4b215..5396ff4 100644
--- a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc
@@ -8,9 +8,9 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-// Unit tests for PayloadSplitter class.
+// Unit tests for RedPayloadSplitter class.
-#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
+#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
#include <assert.h>
@@ -46,7 +46,8 @@
// : |
// | |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-void CreateOpusFecPayload(uint8_t* payload, size_t payload_length,
+void CreateOpusFecPayload(uint8_t* payload,
+ size_t payload_length,
uint8_t payload_value) {
if (payload_length < 2) {
return;
@@ -121,8 +122,10 @@
}
// Create a packet with all payload bytes set to |payload_value|.
-Packet* CreatePacket(uint8_t payload_type, size_t payload_length,
- uint8_t payload_value, bool opus_fec = false) {
+Packet* CreatePacket(uint8_t payload_type,
+ size_t payload_length,
+ uint8_t payload_value,
+ bool opus_fec = false) {
Packet* packet = new Packet;
packet->header.payloadType = payload_type;
packet->header.timestamp = kBaseTimestamp;
@@ -144,22 +147,34 @@
uint16_t sequence_number,
uint32_t timestamp,
uint8_t payload_value,
- bool primary = true) {
+ Packet::Priority priority) {
EXPECT_EQ(payload_length, packet->payload.size());
EXPECT_EQ(payload_type, packet->header.payloadType);
EXPECT_EQ(sequence_number, packet->header.sequenceNumber);
EXPECT_EQ(timestamp, packet->header.timestamp);
- EXPECT_EQ(primary, packet->primary);
+ EXPECT_EQ(priority, packet->priority);
ASSERT_FALSE(packet->payload.empty());
for (size_t i = 0; i < packet->payload.size(); ++i) {
ASSERT_EQ(payload_value, packet->payload.data()[i]);
}
}
+void VerifyPacket(const Packet* packet,
+ size_t payload_length,
+ uint8_t payload_type,
+ uint16_t sequence_number,
+ uint32_t timestamp,
+ uint8_t payload_value,
+ bool primary) {
+ return VerifyPacket(packet, payload_length, payload_type, sequence_number,
+ timestamp, payload_value,
+ Packet::Priority{0, primary ? 0 : 1});
+}
+
// Start of test definitions.
-TEST(PayloadSplitter, CreateAndDestroy) {
- PayloadSplitter* splitter = new PayloadSplitter;
+TEST(RedPayloadSplitter, CreateAndDestroy) {
+ RedPayloadSplitter* splitter = new RedPayloadSplitter;
delete splitter;
}
@@ -170,8 +185,8 @@
Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset);
PacketList packet_list;
packet_list.push_back(packet);
- PayloadSplitter splitter;
- EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list));
+ RedPayloadSplitter splitter;
+ EXPECT_TRUE(splitter.SplitRed(&packet_list));
ASSERT_EQ(2u, packet_list.size());
// Check first packet. The first in list should always be the primary payload.
packet = packet_list.front();
@@ -201,8 +216,8 @@
packet->header.timestamp += kTimestampOffset;
packet->header.sequenceNumber++;
packet_list.push_back(packet);
- PayloadSplitter splitter;
- EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list));
+ RedPayloadSplitter splitter;
+ EXPECT_TRUE(splitter.SplitRed(&packet_list));
ASSERT_EQ(2u, packet_list.size());
// Check first packet.
packet = packet_list.front();
@@ -239,43 +254,43 @@
packet->header.timestamp += kTimestampOffset;
packet->header.sequenceNumber++;
packet_list.push_back(packet);
- PayloadSplitter splitter;
- EXPECT_EQ(PayloadSplitter::kOK, splitter.SplitRed(&packet_list));
+ RedPayloadSplitter splitter;
+ EXPECT_TRUE(splitter.SplitRed(&packet_list));
ASSERT_EQ(6u, packet_list.size());
// Check first packet, A1.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[2], kSequenceNumber,
- kBaseTimestamp, 2, true);
+ kBaseTimestamp, 2, {0, 0});
delete packet;
packet_list.pop_front();
// Check second packet, A2.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[1], kSequenceNumber,
- kBaseTimestamp - kTimestampOffset, 1, false);
+ kBaseTimestamp - kTimestampOffset, 1, {0, 1});
delete packet;
packet_list.pop_front();
// Check third packet, A3.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber,
- kBaseTimestamp - 2 * kTimestampOffset, 0, false);
+ kBaseTimestamp - 2 * kTimestampOffset, 0, {0, 2});
delete packet;
packet_list.pop_front();
// Check fourth packet, B1.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[2], kSequenceNumber + 1,
- kBaseTimestamp + kTimestampOffset, 2, true);
+ kBaseTimestamp + kTimestampOffset, 2, {0, 0});
delete packet;
packet_list.pop_front();
// Check fifth packet, B2.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[1], kSequenceNumber + 1,
- kBaseTimestamp, 1, false);
+ kBaseTimestamp, 1, {0, 1});
delete packet;
packet_list.pop_front();
// Check sixth packet, B3.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber + 1,
- kBaseTimestamp - kTimestampOffset, 0, false);
+ kBaseTimestamp - kTimestampOffset, 0, {0, 2});
delete packet;
}
@@ -305,7 +320,7 @@
decoder_database.RegisterPayload(2, NetEqDecoder::kDecoderAVT, "avt");
decoder_database.RegisterPayload(3, NetEqDecoder::kDecoderILBC, "ilbc");
- PayloadSplitter splitter;
+ RedPayloadSplitter splitter;
splitter.CheckRedPayloads(&packet_list, decoder_database);
ASSERT_EQ(3u, packet_list.size()); // Should have dropped the last packet.
@@ -332,124 +347,13 @@
packet->payload.SetSize(packet->payload.size() - (kPayloadLength + 1));
PacketList packet_list;
packet_list.push_back(packet);
- PayloadSplitter splitter;
- EXPECT_EQ(PayloadSplitter::kRedLengthMismatch,
- splitter.SplitRed(&packet_list));
+ RedPayloadSplitter splitter;
+ EXPECT_FALSE(splitter.SplitRed(&packet_list));
ASSERT_EQ(1u, packet_list.size());
// Check first packet.
packet = packet_list.front();
VerifyPacket(packet, kPayloadLength, payload_types[0], kSequenceNumber,
- kBaseTimestamp - 2 * kTimestampOffset, 0, false);
- delete packet;
- packet_list.pop_front();
-}
-
-TEST(FecPayloadSplitter, MixedPayload) {
- PacketList packet_list;
- DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory());
-
- decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderOpus, "opus");
- decoder_database.RegisterPayload(1, NetEqDecoder::kDecoderPCMu, "pcmu");
-
- Packet* packet = CreatePacket(0, 10, 0xFF, true);
- packet_list.push_back(packet);
-
- packet = CreatePacket(0, 10, 0); // Non-FEC Opus payload.
- packet_list.push_back(packet);
-
- packet = CreatePacket(1, 10, 0); // Non-Opus payload.
- packet_list.push_back(packet);
-
- PayloadSplitter splitter;
- EXPECT_EQ(PayloadSplitter::kOK,
- splitter.SplitFec(&packet_list, &decoder_database));
- EXPECT_EQ(4u, packet_list.size());
-
- // Check first packet.
- packet = packet_list.front();
- EXPECT_EQ(0, packet->header.payloadType);
- EXPECT_EQ(kBaseTimestamp - 20 * 48, packet->header.timestamp);
- EXPECT_EQ(10U, packet->payload.size());
- EXPECT_FALSE(packet->primary);
- delete packet;
- packet_list.pop_front();
-
- // Check second packet.
- packet = packet_list.front();
- EXPECT_EQ(0, packet->header.payloadType);
- EXPECT_EQ(kBaseTimestamp, packet->header.timestamp);
- EXPECT_EQ(10U, packet->payload.size());
- EXPECT_TRUE(packet->primary);
- delete packet;
- packet_list.pop_front();
-
- // Check third packet.
- packet = packet_list.front();
- VerifyPacket(packet, 10, 0, kSequenceNumber, kBaseTimestamp, 0, true);
- delete packet;
- packet_list.pop_front();
-
- // Check fourth packet.
- packet = packet_list.front();
- VerifyPacket(packet, 10, 1, kSequenceNumber, kBaseTimestamp, 0, true);
- delete packet;
-}
-
-TEST(FecPayloadSplitter, EmbedFecInRed) {
- PacketList packet_list;
- DecoderDatabase decoder_database(CreateBuiltinAudioDecoderFactory());
-
- const int kTimestampOffset = 20 * 48; // 20 ms * 48 kHz.
- uint8_t payload_types[] = {0, 0};
- decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderOpus, "opus");
- Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset, true);
- packet_list.push_back(packet);
-
- PayloadSplitter splitter;
- EXPECT_EQ(PayloadSplitter::kOK,
- splitter.SplitRed(&packet_list));
- EXPECT_EQ(PayloadSplitter::kOK,
- splitter.SplitFec(&packet_list, &decoder_database));
-
- EXPECT_EQ(4u, packet_list.size());
-
- // Check first packet. FEC packet copied from primary payload in RED.
- packet = packet_list.front();
- EXPECT_EQ(0, packet->header.payloadType);
- EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp);
- EXPECT_EQ(kPayloadLength, packet->payload.size());
- EXPECT_FALSE(packet->primary);
- EXPECT_EQ(packet->payload[3], 1);
- delete packet;
- packet_list.pop_front();
-
- // Check second packet. Normal packet copied from primary payload in RED.
- packet = packet_list.front();
- EXPECT_EQ(0, packet->header.payloadType);
- EXPECT_EQ(kBaseTimestamp, packet->header.timestamp);
- EXPECT_EQ(kPayloadLength, packet->payload.size());
- EXPECT_TRUE(packet->primary);
- EXPECT_EQ(packet->payload[3], 1);
- delete packet;
- packet_list.pop_front();
-
- // Check third packet. FEC packet copied from secondary payload in RED.
- packet = packet_list.front();
- EXPECT_EQ(0, packet->header.payloadType);
- EXPECT_EQ(kBaseTimestamp - 2 * kTimestampOffset, packet->header.timestamp);
- EXPECT_EQ(kPayloadLength, packet->payload.size());
- EXPECT_FALSE(packet->primary);
- EXPECT_EQ(packet->payload[3], 0);
- delete packet;
- packet_list.pop_front();
-
- // Check fourth packet. Normal packet copied from primary payload in RED.
- packet = packet_list.front();
- EXPECT_EQ(0, packet->header.payloadType);
- EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp);
- EXPECT_EQ(kPayloadLength, packet->payload.size());
- EXPECT_TRUE(packet->primary);
- EXPECT_EQ(packet->payload[3], 0);
+ kBaseTimestamp - 2 * kTimestampOffset, 0, {0, 2});
delete packet;
packet_list.pop_front();
}
