| /* |
| * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "webrtc/modules/audio_coding/neteq/payload_splitter.h" |
| |
| #include <assert.h> |
| |
| #include "webrtc/base/checks.h" |
| #include "webrtc/base/logging.h" |
| #include "webrtc/modules/audio_coding/neteq/decoder_database.h" |
| |
| namespace webrtc { |
| |
| // The method loops through a list of packets {A, B, C, ...}. Each packet is |
| // split into its corresponding RED payloads, {A1, A2, ...}, which is |
| // temporarily held in the list |new_packets|. |
| // When the first packet in |packet_list| has been processed, the orignal packet |
| // is replaced by the new ones in |new_packets|, so that |packet_list| becomes: |
| // {A1, A2, ..., B, C, ...}. The method then continues with B, and C, until all |
| // the original packets have been replaced by their split payloads. |
| int PayloadSplitter::SplitRed(PacketList* packet_list) { |
| int ret = kOK; |
| PacketList::iterator it = packet_list->begin(); |
| while (it != packet_list->end()) { |
| const Packet* red_packet = (*it); |
| assert(!red_packet->payload.empty()); |
| const uint8_t* payload_ptr = red_packet->payload.data(); |
| |
| // Read RED headers (according to RFC 2198): |
| // |
| // 0 1 2 3 |
| // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 |
| // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| // |F| block PT | timestamp offset | block length | |
| // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| // Last RED header: |
| // 0 1 2 3 4 5 6 7 |
| // +-+-+-+-+-+-+-+-+ |
| // |0| Block PT | |
| // +-+-+-+-+-+-+-+-+ |
| |
| struct RedHeader { |
| uint8_t payload_type; |
| uint32_t timestamp; |
| size_t payload_length; |
| bool primary; |
| }; |
| |
| std::vector<RedHeader> new_headers; |
| bool last_block = false; |
| size_t sum_length = 0; |
| while (!last_block) { |
| RedHeader new_header; |
| // Check the F bit. If F == 0, this was the last block. |
| last_block = ((*payload_ptr & 0x80) == 0); |
| // Bits 1 through 7 are payload type. |
| new_header.payload_type = payload_ptr[0] & 0x7F; |
| if (last_block) { |
| // No more header data to read. |
| ++sum_length; // Account for RED header size of 1 byte. |
| new_header.timestamp = red_packet->header.timestamp; |
| new_header.payload_length = red_packet->payload.size() - sum_length; |
| new_header.primary = true; // Last block is always primary. |
| payload_ptr += 1; // Advance to first payload byte. |
| } else { |
| // Bits 8 through 21 are timestamp offset. |
| int timestamp_offset = (payload_ptr[1] << 6) + |
| ((payload_ptr[2] & 0xFC) >> 2); |
| new_header.timestamp = red_packet->header.timestamp - timestamp_offset; |
| // Bits 22 through 31 are payload length. |
| new_header.payload_length = |
| ((payload_ptr[2] & 0x03) << 8) + payload_ptr[3]; |
| new_header.primary = false; |
| payload_ptr += 4; // Advance to next RED header. |
| } |
| sum_length += new_header.payload_length; |
| sum_length += 4; // Account for RED header size of 4 bytes. |
| // Store in new list of packets. |
| new_headers.push_back(new_header); |
| } |
| |
| // Populate the new packets with payload data. |
| // |payload_ptr| now points at the first payload byte. |
| PacketList new_packets; // An empty list to store the split packets in. |
| for (const auto& new_header : new_headers) { |
| size_t payload_length = new_header.payload_length; |
| if (payload_ptr + payload_length > |
| red_packet->payload.data() + red_packet->payload.size()) { |
| // The block lengths in the RED headers do not match the overall packet |
| // length. Something is corrupt. Discard this and the remaining |
| // payloads from this packet. |
| LOG(LS_WARNING) << "SplitRed length mismatch"; |
| ret = kRedLengthMismatch; |
| break; |
| } |
| Packet* new_packet = new Packet; |
| new_packet->header = red_packet->header; |
| new_packet->header.timestamp = new_header.timestamp; |
| new_packet->header.payloadType = new_header.payload_type; |
| new_packet->primary = new_header.primary; |
| new_packet->payload.SetData(payload_ptr, payload_length); |
| new_packets.push_front(new_packet); |
| payload_ptr += payload_length; |
| } |
| // Insert new packets into original list, before the element pointed to by |
| // iterator |it|. |
| packet_list->splice(it, new_packets, new_packets.begin(), |
| new_packets.end()); |
| // Delete old packet payload. |
| delete (*it); |
| // Remove |it| from the packet list. This operation effectively moves the |
| // iterator |it| to the next packet in the list. Thus, we do not have to |
| // increment it manually. |
| it = packet_list->erase(it); |
| } |
| return ret; |
| } |
| |
| int PayloadSplitter::SplitFec(PacketList* packet_list, |
| DecoderDatabase* decoder_database) { |
| PacketList::iterator it = packet_list->begin(); |
| // Iterate through all packets in |packet_list|. |
| while (it != packet_list->end()) { |
| Packet* packet = (*it); // Just to make the notation more intuitive. |
| // Get codec type for this payload. |
| uint8_t payload_type = packet->header.payloadType; |
| const DecoderDatabase::DecoderInfo* info = |
| decoder_database->GetDecoderInfo(payload_type); |
| if (!info) { |
| LOG(LS_WARNING) << "SplitFec unknown payload type"; |
| return kUnknownPayloadType; |
| } |
| // No splitting for a sync-packet. |
| if (packet->sync_packet) { |
| ++it; |
| continue; |
| } |
| |
| // Not an FEC packet. |
| AudioDecoder* decoder = decoder_database->GetDecoder(payload_type); |
| // decoder should not return NULL, except for comfort noise payloads which |
| // are handled separately. |
| assert(decoder != NULL || decoder_database->IsComfortNoise(payload_type)); |
| if (!decoder || |
| !decoder->PacketHasFec(packet->payload.data(), |
| packet->payload.size())) { |
| ++it; |
| continue; |
| } |
| |
| switch (info->codec_type) { |
| case NetEqDecoder::kDecoderOpus: |
| case NetEqDecoder::kDecoderOpus_2ch: { |
| // The main payload of this packet should be decoded as a primary |
| // payload, even if it comes as a secondary payload in a RED packet. |
| packet->primary = true; |
| |
| Packet* new_packet = new Packet; |
| new_packet->header = packet->header; |
| int duration = decoder->PacketDurationRedundant(packet->payload.data(), |
| packet->payload.size()); |
| new_packet->header.timestamp -= duration; |
| new_packet->payload.SetData(packet->payload); |
| new_packet->primary = false; |
| new_packet->sync_packet = packet->sync_packet; |
| // Waiting time should not be set here. |
| RTC_DCHECK(!packet->waiting_time); |
| |
| packet_list->insert(it, new_packet); |
| break; |
| } |
| default: { |
| LOG(LS_WARNING) << "SplitFec wrong payload type"; |
| return kFecSplitError; |
| } |
| } |
| |
| ++it; |
| } |
| return kOK; |
| } |
| |
| int PayloadSplitter::CheckRedPayloads(PacketList* packet_list, |
| const DecoderDatabase& decoder_database) { |
| PacketList::iterator it = packet_list->begin(); |
| int main_payload_type = -1; |
| int num_deleted_packets = 0; |
| while (it != packet_list->end()) { |
| uint8_t this_payload_type = (*it)->header.payloadType; |
| if (!decoder_database.IsDtmf(this_payload_type) && |
| !decoder_database.IsComfortNoise(this_payload_type)) { |
| if (main_payload_type == -1) { |
| // This is the first packet in the list which is non-DTMF non-CNG. |
| main_payload_type = this_payload_type; |
| } else { |
| if (this_payload_type != main_payload_type) { |
| // We do not allow redundant payloads of a different type. |
| // Discard this payload. |
| delete (*it); |
| // Remove |it| from the packet list. This operation effectively |
| // moves the iterator |it| to the next packet in the list. Thus, we |
| // do not have to increment it manually. |
| it = packet_list->erase(it); |
| ++num_deleted_packets; |
| continue; |
| } |
| } |
| } |
| ++it; |
| } |
| return num_deleted_packets; |
| } |
| |
| int PayloadSplitter::SplitAudio(PacketList* packet_list, |
| const DecoderDatabase& decoder_database) { |
| PacketList::iterator it = packet_list->begin(); |
| // Iterate through all packets in |packet_list|. |
| while (it != packet_list->end()) { |
| Packet* packet = (*it); // Just to make the notation more intuitive. |
| // Get codec type for this payload. |
| const DecoderDatabase::DecoderInfo* info = |
| decoder_database.GetDecoderInfo(packet->header.payloadType); |
| if (!info) { |
| LOG(LS_WARNING) << "SplitAudio unknown payload type"; |
| return kUnknownPayloadType; |
| } |
| // No splitting for a sync-packet. |
| if (packet->sync_packet) { |
| ++it; |
| continue; |
| } |
| PacketList new_packets; |
| switch (info->codec_type) { |
| case NetEqDecoder::kDecoderPCMu: |
| case NetEqDecoder::kDecoderPCMa: { |
| // 8 bytes per ms; 8 timestamps per ms. |
| SplitBySamples(packet, 8, 8, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCMu_2ch: |
| case NetEqDecoder::kDecoderPCMa_2ch: { |
| // 2 * 8 bytes per ms; 8 timestamps per ms. |
| SplitBySamples(packet, 2 * 8, 8, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderG722: { |
| // 8 bytes per ms; 16 timestamps per ms. |
| SplitBySamples(packet, 8, 16, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16B: { |
| // 16 bytes per ms; 8 timestamps per ms. |
| SplitBySamples(packet, 16, 8, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16Bwb: { |
| // 32 bytes per ms; 16 timestamps per ms. |
| SplitBySamples(packet, 32, 16, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16Bswb32kHz: { |
| // 64 bytes per ms; 32 timestamps per ms. |
| SplitBySamples(packet, 64, 32, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16Bswb48kHz: { |
| // 96 bytes per ms; 48 timestamps per ms. |
| SplitBySamples(packet, 96, 48, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16B_2ch: { |
| // 2 * 16 bytes per ms; 8 timestamps per ms. |
| SplitBySamples(packet, 2 * 16, 8, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16Bwb_2ch: { |
| // 2 * 32 bytes per ms; 16 timestamps per ms. |
| SplitBySamples(packet, 2 * 32, 16, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch: { |
| // 2 * 64 bytes per ms; 32 timestamps per ms. |
| SplitBySamples(packet, 2 * 64, 32, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch: { |
| // 2 * 96 bytes per ms; 48 timestamps per ms. |
| SplitBySamples(packet, 2 * 96, 48, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderPCM16B_5ch: { |
| // 5 * 16 bytes per ms; 8 timestamps per ms. |
| SplitBySamples(packet, 5 * 16, 8, &new_packets); |
| break; |
| } |
| case NetEqDecoder::kDecoderILBC: { |
| size_t bytes_per_frame; |
| int timestamps_per_frame; |
| if (packet->payload.size() >= 950) { |
| LOG(LS_WARNING) << "SplitAudio too large iLBC payload"; |
| return kTooLargePayload; |
| } |
| if (packet->payload.size() % 38 == 0) { |
| // 20 ms frames. |
| bytes_per_frame = 38; |
| timestamps_per_frame = 160; |
| } else if (packet->payload.size() % 50 == 0) { |
| // 30 ms frames. |
| bytes_per_frame = 50; |
| timestamps_per_frame = 240; |
| } else { |
| LOG(LS_WARNING) << "SplitAudio invalid iLBC payload"; |
| return kFrameSplitError; |
| } |
| int ret = SplitByFrames(packet, bytes_per_frame, timestamps_per_frame, |
| &new_packets); |
| if (ret < 0) { |
| return ret; |
| } else if (ret == kNoSplit) { |
| // Do not split at all. Simply advance to the next packet in the list. |
| ++it; |
| // We do not have any new packets to insert, and should not delete the |
| // old one. Skip the code after the switch case, and jump straight to |
| // the next packet in the while loop. |
| continue; |
| } |
| break; |
| } |
| default: { |
| // Do not split at all. Simply advance to the next packet in the list. |
| ++it; |
| // We do not have any new packets to insert, and should not delete the |
| // old one. Skip the code after the switch case, and jump straight to |
| // the next packet in the while loop. |
| continue; |
| } |
| } |
| // Insert new packets into original list, before the element pointed to by |
| // iterator |it|. |
| packet_list->splice(it, new_packets, new_packets.begin(), |
| new_packets.end()); |
| // Delete old packet payload. |
| delete (*it); |
| // Remove |it| from the packet list. This operation effectively moves the |
| // iterator |it| to the next packet in the list. Thus, we do not have to |
| // increment it manually. |
| it = packet_list->erase(it); |
| } |
| return kOK; |
| } |
| |
| void PayloadSplitter::SplitBySamples(const Packet* packet, |
| size_t bytes_per_ms, |
| uint32_t timestamps_per_ms, |
| PacketList* new_packets) { |
| assert(packet); |
| assert(new_packets); |
| |
| size_t split_size_bytes = packet->payload.size(); |
| |
| // Find a "chunk size" >= 20 ms and < 40 ms. |
| size_t min_chunk_size = bytes_per_ms * 20; |
| // Reduce the split size by half as long as |split_size_bytes| is at least |
| // twice the minimum chunk size (so that the resulting size is at least as |
| // large as the minimum chunk size). |
| while (split_size_bytes >= 2 * min_chunk_size) { |
| split_size_bytes >>= 1; |
| } |
| uint32_t timestamps_per_chunk = static_cast<uint32_t>( |
| split_size_bytes * timestamps_per_ms / bytes_per_ms); |
| uint32_t timestamp = packet->header.timestamp; |
| |
| const uint8_t* payload_ptr = packet->payload.data(); |
| size_t len = packet->payload.size(); |
| while (len >= (2 * split_size_bytes)) { |
| Packet* new_packet = new Packet; |
| new_packet->header = packet->header; |
| new_packet->header.timestamp = timestamp; |
| timestamp += timestamps_per_chunk; |
| new_packet->primary = packet->primary; |
| new_packet->payload.SetData(payload_ptr, split_size_bytes); |
| payload_ptr += split_size_bytes; |
| new_packets->push_back(new_packet); |
| len -= split_size_bytes; |
| } |
| |
| if (len > 0) { |
| Packet* new_packet = new Packet; |
| new_packet->header = packet->header; |
| new_packet->header.timestamp = timestamp; |
| new_packet->primary = packet->primary; |
| new_packet->payload.SetData(payload_ptr, len); |
| new_packets->push_back(new_packet); |
| } |
| } |
| |
| int PayloadSplitter::SplitByFrames(const Packet* packet, |
| size_t bytes_per_frame, |
| uint32_t timestamps_per_frame, |
| PacketList* new_packets) { |
| if (packet->payload.size() % bytes_per_frame != 0) { |
| LOG(LS_WARNING) << "SplitByFrames length mismatch"; |
| return kFrameSplitError; |
| } |
| |
| if (packet->payload.size() == bytes_per_frame) { |
| // Special case. Do not split the payload. |
| return kNoSplit; |
| } |
| |
| uint32_t timestamp = packet->header.timestamp; |
| const uint8_t* payload_ptr = packet->payload.data(); |
| size_t len = packet->payload.size(); |
| while (len > 0) { |
| assert(len >= bytes_per_frame); |
| Packet* new_packet = new Packet; |
| new_packet->header = packet->header; |
| new_packet->header.timestamp = timestamp; |
| timestamp += timestamps_per_frame; |
| new_packet->primary = packet->primary; |
| new_packet->payload.SetData(payload_ptr, bytes_per_frame); |
| payload_ptr += bytes_per_frame; |
| new_packets->push_back(new_packet); |
| len -= bytes_per_frame; |
| } |
| return kOK; |
| } |
| |
| } // namespace webrtc |