H264SpsPpsTracker class which keep tracks of SPS/PPS.
The H264SpsPpsTracker class:
- Keeps track of all received SPS/PPS.
- Decides whether a packet should be inserted into the PacketBuffer or not.
- Don't insert if this packet only contains SPS and/or PPS.
- Don't insert if this is the first packet of and IDR and we have not
received the required SPS/PPS.
- Insert start codes, and in the case of the first packet of an IDR prepend
the bitstream with the given SPS/PPS for this IDR.
BUG=webrtc:5514
Review-Url: https://codereview.webrtc.org/2466993003
Cr-Commit-Position: refs/heads/master@{#14906}
diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn
index fa58999..a7ea119 100644
--- a/webrtc/modules/BUILD.gn
+++ b/webrtc/modules/BUILD.gn
@@ -491,6 +491,7 @@
"video_coding/codecs/vp8/simulcast_unittest.h",
"video_coding/decoding_state_unittest.cc",
"video_coding/frame_buffer2_unittest.cc",
+ "video_coding/h264_sps_pps_tracker_unittest.cc",
"video_coding/histogram_unittest.cc",
"video_coding/include/mock/mock_vcm_callbacks.h",
"video_coding/jitter_buffer_unittest.cc",
diff --git a/webrtc/modules/include/module_common_types.h b/webrtc/modules/include/module_common_types.h
index 4e43110..a5ea5c8 100644
--- a/webrtc/modules/include/module_common_types.h
+++ b/webrtc/modules/include/module_common_types.h
@@ -264,6 +264,10 @@
uint8_t type;
int sps_id;
int pps_id;
+
+ // Offset and size are only valid for non-FuA packets.
+ size_t offset;
+ size_t size;
};
const size_t kMaxNalusPerPacket = 10;
diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc
index 2747945..b32e78e 100644
--- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc
+++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc
@@ -411,6 +411,8 @@
NaluInfo nalu;
nalu.type = payload_data[start_offset] & kTypeMask;
+ nalu.offset = start_offset;
+ nalu.size = end_offset - start_offset;
nalu.sps_id = -1;
nalu.pps_id = -1;
start_offset += H264::kNaluTypeSize;
diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn
index a21a79b..e567b89 100644
--- a/webrtc/modules/video_coding/BUILD.gn
+++ b/webrtc/modules/video_coding/BUILD.gn
@@ -29,6 +29,8 @@
"generic_decoder.h",
"generic_encoder.cc",
"generic_encoder.h",
+ "h264_sps_pps_tracker.cc",
+ "h264_sps_pps_tracker.h",
"histogram.cc",
"histogram.h",
"include/video_coding.h",
diff --git a/webrtc/modules/video_coding/h264_sps_pps_tracker.cc b/webrtc/modules/video_coding/h264_sps_pps_tracker.cc
new file mode 100644
index 0000000..b44f477
--- /dev/null
+++ b/webrtc/modules/video_coding/h264_sps_pps_tracker.cc
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h"
+
+#include <string>
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/common_video/h264/h264_common.h"
+#include "webrtc/modules/video_coding/frame_object.h"
+#include "webrtc/modules/video_coding/packet_buffer.h"
+
+namespace webrtc {
+namespace video_coding {
+
+namespace {
+const uint8_t start_code_h264[] = {0, 0, 0, 1};
+} // namespace
+
+bool H264SpsPpsTracker::CopyAndFixBitstream(VCMPacket* packet) {
+ RTC_DCHECK(packet->codec == kVideoCodecH264);
+
+ const uint8_t* data = packet->dataPtr;
+ const size_t data_size = packet->sizeBytes;
+ const RTPVideoHeader& video_header = packet->video_header;
+ const RTPVideoHeaderH264& codec_header = video_header.codecHeader.H264;
+
+ // Packets that only contains SPS/PPS are not decodable by themselves, and
+ // to avoid frames being created containing only these two nalus we don't
+ // insert them into the PacketBuffer. Instead we save the SPS/PPS and
+ // prepend the bitstream of first packet of an IDR referring to the
+ // corresponding SPS/PPS id.
+ bool insert_packet = codec_header.nalus_length == 0 ? true : false;
+
+ int pps_id = -1;
+ size_t required_size = 0;
+ for (size_t i = 0; i < codec_header.nalus_length; ++i) {
+ const NaluInfo& nalu = codec_header.nalus[i];
+ switch (nalu.type) {
+ case H264::NaluType::kSps: {
+ // Save SPS.
+ sps_data_[nalu.sps_id].size = nalu.size;
+ sps_data_[nalu.sps_id].data.reset(new uint8_t[nalu.size]);
+ memcpy(sps_data_[nalu.sps_id].data.get(), data + nalu.offset,
+ nalu.size);
+ break;
+ }
+ case H264::NaluType::kPps: {
+ // Save PPS.
+ pps_data_[nalu.pps_id].sps_id = nalu.sps_id;
+ pps_data_[nalu.pps_id].size = nalu.size;
+ pps_data_[nalu.pps_id].data.reset(new uint8_t[nalu.size]);
+ memcpy(pps_data_[nalu.pps_id].data.get(), data + nalu.offset,
+ nalu.size);
+ break;
+ }
+ case H264::NaluType::kIdr: {
+ // If this is the first packet of an IDR, make sure we have the required
+ // SPS/PPS and also calculate how much extra space we need in the buffer
+ // to prepend the SPS/PPS to the bitstream with start codes.
+ if (video_header.isFirstPacket) {
+ if (nalu.pps_id == -1) {
+ LOG(LS_WARNING) << "No PPS id in IDR nalu.";
+ return false;
+ }
+
+ auto pps = pps_data_.find(nalu.pps_id);
+ if (pps == pps_data_.end()) {
+ LOG(LS_WARNING) << "No PPS with id << " << nalu.pps_id
+ << " received";
+ return false;
+ }
+
+ auto sps = sps_data_.find(pps->second.sps_id);
+ if (sps == sps_data_.end()) {
+ LOG(LS_WARNING) << "No SPS with id << "
+ << pps_data_[nalu.pps_id].sps_id << " received";
+ return false;
+ }
+
+ pps_id = nalu.pps_id;
+ required_size += pps->second.size + sizeof(start_code_h264);
+ required_size += sps->second.size + sizeof(start_code_h264);
+ }
+ FALLTHROUGH();
+ }
+ default: {
+ // Something other than an SPS/PPS nalu in this packet, then it should
+ // be inserted into the PacketBuffer.
+ insert_packet = true;
+ }
+ }
+ }
+
+ if (!insert_packet)
+ return false;
+
+ // Calculate how much space we need for the rest of the bitstream.
+ if (codec_header.packetization_type == kH264StapA) {
+ const uint8_t* nalu_ptr = data + 1;
+ while (nalu_ptr < data + data_size) {
+ RTC_DCHECK(video_header.isFirstPacket);
+ required_size += sizeof(start_code_h264);
+
+ // The first two bytes describe the length of a segment.
+ uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
+ nalu_ptr += 2;
+
+ required_size += segment_length;
+ nalu_ptr += segment_length;
+ }
+ } else {
+ if (video_header.isFirstPacket)
+ required_size += sizeof(start_code_h264);
+ required_size += data_size;
+ }
+
+ // Then we copy to the new buffer.
+ uint8_t* buffer = new uint8_t[required_size];
+ uint8_t* insert_at = buffer;
+
+ // If pps_id != -1 then we have the SPS/PPS and they should be prepended
+ // to the bitstream with start codes inserted.
+ if (pps_id != -1) {
+ // Insert SPS.
+ memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+ insert_at += sizeof(start_code_h264);
+ memcpy(insert_at, sps_data_[pps_data_[pps_id].sps_id].data.get(),
+ sps_data_[pps_data_[pps_id].sps_id].size);
+ insert_at += sps_data_[pps_data_[pps_id].sps_id].size;
+
+ // Insert PPS.
+ memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+ insert_at += sizeof(start_code_h264);
+ memcpy(insert_at, pps_data_[pps_id].data.get(), pps_data_[pps_id].size);
+ insert_at += pps_data_[pps_id].size;
+ }
+
+ // Copy the rest of the bitstream and insert start codes.
+ if (codec_header.packetization_type == kH264StapA) {
+ const uint8_t* nalu_ptr = data + 1;
+ while (nalu_ptr < data + data_size) {
+ memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+ insert_at += sizeof(start_code_h264);
+
+ // The first two bytes describe the length of a segment.
+ uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
+ nalu_ptr += 2;
+
+ size_t copy_end = nalu_ptr - data + segment_length;
+ if (copy_end > data_size) {
+ delete[] buffer;
+ return false;
+ }
+
+ memcpy(insert_at, nalu_ptr, segment_length);
+ insert_at += segment_length;
+ nalu_ptr += segment_length;
+ }
+ } else {
+ if (video_header.isFirstPacket) {
+ memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+ insert_at += sizeof(start_code_h264);
+ }
+ memcpy(insert_at, data, data_size);
+ }
+
+ packet->dataPtr = buffer;
+ packet->sizeBytes = required_size;
+ return true;
+}
+
+} // namespace video_coding
+} // namespace webrtc
diff --git a/webrtc/modules/video_coding/h264_sps_pps_tracker.h b/webrtc/modules/video_coding/h264_sps_pps_tracker.h
new file mode 100644
index 0000000..6de092a
--- /dev/null
+++ b/webrtc/modules/video_coding/h264_sps_pps_tracker.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_
+#define WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_
+
+#include <cstdint>
+#include <map>
+#include <memory>
+
+#include "webrtc/modules/include/module_common_types.h"
+
+namespace webrtc {
+
+class VCMPacket;
+
+namespace video_coding {
+
+class H264SpsPpsTracker {
+ public:
+ bool CopyAndFixBitstream(VCMPacket* packet);
+
+ private:
+ struct PpsInfo {
+ int sps_id = -1;
+ size_t size = 0;
+ std::unique_ptr<uint8_t[]> data;
+ };
+
+ struct SpsInfo {
+ size_t size = 0;
+ std::unique_ptr<uint8_t[]> data;
+ };
+
+ std::map<uint32_t, PpsInfo> pps_data_;
+ std::map<uint32_t, SpsInfo> sps_data_;
+};
+
+} // namespace video_coding
+} // namespace webrtc
+
+#endif // WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_
diff --git a/webrtc/modules/video_coding/h264_sps_pps_tracker_unittest.cc b/webrtc/modules/video_coding/h264_sps_pps_tracker_unittest.cc
new file mode 100644
index 0000000..f467028
--- /dev/null
+++ b/webrtc/modules/video_coding/h264_sps_pps_tracker_unittest.cc
@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h"
+
+#include <vector>
+
+#include "webrtc/modules/video_coding/packet.h"
+#include "webrtc/test/gtest.h"
+#include "webrtc/common_video/h264/h264_common.h"
+
+namespace webrtc {
+namespace video_coding {
+
+namespace {
+const uint8_t start_code[] = {0, 0, 0, 1};
+} // namespace
+
+class TestH264SpsPpsTracker : public ::testing::Test {
+ public:
+ VCMPacket GetDefaultPacket() {
+ VCMPacket packet;
+ packet.codec = kVideoCodecH264;
+ packet.video_header.codecHeader.H264.nalus_length = 0;
+ packet.video_header.isFirstPacket = false;
+ packet.video_header.codecHeader.H264.packetization_type = kH264SingleNalu;
+
+ return packet;
+ }
+
+ void AddSps(VCMPacket* packet, int sps_id, std::vector<uint8_t>* data) {
+ NaluInfo info;
+ info.type = H264::NaluType::kSps;
+ info.sps_id = sps_id;
+ info.pps_id = -1;
+ info.offset = data->size();
+ info.size = 2;
+ data->push_back(H264::NaluType::kSps);
+ data->push_back(sps_id); // The sps data, just a single byte.
+
+ packet->video_header.codecHeader.H264
+ .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info;
+ }
+
+ void AddPps(VCMPacket* packet,
+ int sps_id,
+ int pps_id,
+ std::vector<uint8_t>* data) {
+ NaluInfo info;
+ info.type = H264::NaluType::kPps;
+ info.sps_id = sps_id;
+ info.pps_id = pps_id;
+ info.offset = data->size();
+ info.size = 2;
+ data->push_back(H264::NaluType::kPps);
+ data->push_back(pps_id); // The pps data, just a single byte.
+
+ packet->video_header.codecHeader.H264
+ .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info;
+ }
+
+ void AddIdr(VCMPacket* packet, int pps_id) {
+ NaluInfo info;
+ info.type = H264::NaluType::kIdr;
+ info.sps_id = -1;
+ info.pps_id = pps_id;
+
+ packet->video_header.codecHeader.H264
+ .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info;
+ }
+
+ protected:
+ H264SpsPpsTracker tracker_;
+};
+
+TEST_F(TestH264SpsPpsTracker, NoNalus) {
+ uint8_t data[] = {1, 2, 3};
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.codecHeader.H264.packetization_type = kH264FuA;
+ packet.dataPtr = data;
+ packet.sizeBytes = sizeof(data);
+
+ EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+ EXPECT_EQ(memcmp(packet.dataPtr, data, sizeof(data)), 0);
+ delete[] packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, FuAFirstPacket) {
+ uint8_t data[] = {1, 2, 3};
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.codecHeader.H264.packetization_type = kH264FuA;
+ packet.video_header.isFirstPacket = true;
+ packet.dataPtr = data;
+ packet.sizeBytes = sizeof(data);
+
+ EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+ std::vector<uint8_t> expected;
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {1, 2, 3});
+ EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0);
+ delete[] packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, StapAIncorrectSegmentLength) {
+ uint8_t data[] = {0, 0, 2, 0};
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.codecHeader.H264.packetization_type = kH264StapA;
+ packet.video_header.isFirstPacket = true;
+ packet.dataPtr = data;
+ packet.sizeBytes = sizeof(data);
+
+ EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet));
+}
+
+TEST_F(TestH264SpsPpsTracker, NoNalusFirstPacket) {
+ uint8_t data[] = {1, 2, 3};
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.isFirstPacket = true;
+ packet.dataPtr = data;
+ packet.sizeBytes = sizeof(data);
+
+ EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+ std::vector<uint8_t> expected;
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {1, 2, 3});
+ EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0);
+ delete[] packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, IdrNoSpsPpsInserted) {
+ std::vector<uint8_t> data = {1, 2, 3};
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.codecHeader.H264.packetization_type = kH264FuA;
+
+ AddIdr(&packet, 0);
+ packet.dataPtr = data.data();
+ packet.sizeBytes = data.size();
+
+ EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+ EXPECT_EQ(memcmp(packet.dataPtr, data.data(), data.size()), 0);
+ delete[] packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoSpsPpsInserted) {
+ std::vector<uint8_t> data = {1, 2, 3};
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.isFirstPacket = true;
+
+ AddIdr(&packet, 0);
+ packet.dataPtr = data.data();
+ packet.sizeBytes = data.size();
+
+ EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet));
+}
+
+TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoPpsInserted) {
+ std::vector<uint8_t> data = {1, 2, 3};
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.isFirstPacket = true;
+
+ AddSps(&packet, 0, &data);
+ AddIdr(&packet, 0);
+ packet.dataPtr = data.data();
+ packet.sizeBytes = data.size();
+
+ EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet));
+}
+
+TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoSpsInserted) {
+ std::vector<uint8_t> data = {1, 2, 3};
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.isFirstPacket = true;
+
+ AddPps(&packet, 0, 0, &data);
+ AddIdr(&packet, 0);
+ packet.dataPtr = data.data();
+ packet.sizeBytes = data.size();
+
+ EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet));
+}
+
+TEST_F(TestH264SpsPpsTracker, SpsPpsPacketThenIdrFirstPacket) {
+ std::vector<uint8_t> data;
+ VCMPacket sps_pps_packet = GetDefaultPacket();
+
+ // Insert SPS/PPS
+ AddSps(&sps_pps_packet, 0, &data);
+ AddPps(&sps_pps_packet, 0, 1, &data);
+ sps_pps_packet.dataPtr = data.data();
+ sps_pps_packet.sizeBytes = data.size();
+ EXPECT_FALSE(tracker_.CopyAndFixBitstream(&sps_pps_packet));
+ data.clear();
+
+ // Insert first packet of the IDR
+ VCMPacket idr_packet = GetDefaultPacket();
+ idr_packet.video_header.isFirstPacket = true;
+ AddIdr(&idr_packet, 1);
+ data.insert(data.end(), {1, 2, 3});
+ idr_packet.dataPtr = data.data();
+ idr_packet.sizeBytes = data.size();
+ EXPECT_TRUE(tracker_.CopyAndFixBitstream(&idr_packet));
+
+ std::vector<uint8_t> expected;
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {H264::NaluType::kSps, 0});
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {H264::NaluType::kPps, 1});
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {1, 2, 3});
+ EXPECT_EQ(memcmp(idr_packet.dataPtr, expected.data(), expected.size()), 0);
+ delete[] idr_packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, SpsPpsIdrInStapA) {
+ std::vector<uint8_t> data;
+ VCMPacket packet = GetDefaultPacket();
+ packet.video_header.codecHeader.H264.packetization_type = kH264StapA;
+ packet.video_header.isFirstPacket = true; // Always true for StapA
+
+ data.insert(data.end(), {0}); // First byte is ignored
+ data.insert(data.end(), {0, 2}); // Length of segment
+ AddSps(&packet, 13, &data);
+ data.insert(data.end(), {0, 2}); // Length of segment
+ AddPps(&packet, 13, 27, &data);
+ data.insert(data.end(), {0, 5}); // Length of segment
+ AddIdr(&packet, 27);
+ data.insert(data.end(), {1, 2, 3, 2, 1});
+
+ packet.dataPtr = data.data();
+ packet.sizeBytes = data.size();
+ EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+
+ std::vector<uint8_t> expected;
+ // The SPS/PPS is repeated because this packet both contains the SPS/PPS
+ // and it is the first packet of an IDR, which will cause the SPS/PPS to be
+ // prepended to the bitstream.
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {H264::NaluType::kSps, 13});
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {H264::NaluType::kPps, 27});
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {H264::NaluType::kSps, 13});
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {H264::NaluType::kPps, 27});
+ expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+ expected.insert(expected.end(), {1, 2, 3, 2, 1});
+
+ EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0);
+ delete[] packet.dataPtr;
+}
+
+} // namespace video_coding
+} // namespace webrtc
