H26xPacketBuffer handles out of band H.264 parameter sets.
This CL updates H26xPacketBuffer to store and prepend SPS and PPS for
H.264 bitstreams when IDR only keyframe is allowed.
Bug: webrtc:13485
Change-Id: Ic1edc623dff568d54d3ce29b42dd8eab3312f5cb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/342225
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41986}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 4da259e..ef63d2d 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -130,6 +130,7 @@
]
deps = [
":codec_globals_headers",
+ ":h264_sprop_parameter_sets",
":packet_buffer",
"../../api:array_view",
"../../api:rtp_packet_info",
@@ -162,6 +163,17 @@
absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ]
}
+rtc_library("h264_sprop_parameter_sets") {
+ sources = [
+ "h264_sprop_parameter_sets.cc",
+ "h264_sprop_parameter_sets.h",
+ ]
+ deps = [
+ "../../rtc_base:logging",
+ "../../rtc_base/third_party/base64",
+ ]
+}
+
rtc_library("video_coding") {
visibility = [ "*" ]
sources = [
@@ -172,8 +184,6 @@
"fec_rate_table.h",
"generic_decoder.cc",
"generic_decoder.h",
- "h264_sprop_parameter_sets.cc",
- "h264_sprop_parameter_sets.h",
"h264_sps_pps_tracker.cc",
"h264_sps_pps_tracker.h",
"include/video_codec_initializer.h",
@@ -203,6 +213,7 @@
":codec_globals_headers",
":encoded_frame",
":frame_helpers",
+ ":h264_sprop_parameter_sets",
":video_codec_interface",
":video_coding_utility",
":webrtc_vp8_scalability",
@@ -1173,6 +1184,7 @@
":encoded_frame",
":frame_dependencies_calculator",
":frame_helpers",
+ ":h264_sprop_parameter_sets",
":h26x_packet_buffer",
":nack_requester",
":packet_buffer",
diff --git a/modules/video_coding/h26x_packet_buffer.cc b/modules/video_coding/h26x_packet_buffer.cc
index bca2b5c..133ab47 100644
--- a/modules/video_coding/h26x_packet_buffer.cc
+++ b/modules/video_coding/h26x_packet_buffer.cc
@@ -19,10 +19,13 @@
#include "api/rtp_packet_info.h"
#include "api/video/video_frame_type.h"
#include "common_video/h264/h264_common.h"
+#include "common_video/h264/pps_parser.h"
+#include "common_video/h264/sps_parser.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
#include "modules/video_coding/codecs/h264/include/h264_globals.h"
+#include "modules/video_coding/h264_sprop_parameter_sets.h"
#include "rtc_base/checks.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/logging.h"
@@ -91,55 +94,6 @@
}
#endif
-// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to
-// fiddle with the payload at this point.
-rtc::CopyOnWriteBuffer FixH264VideoPayload(
- rtc::ArrayView<const uint8_t> payload,
- const RTPVideoHeader& video_header) {
- constexpr uint8_t kStartCode[] = {0, 0, 0, 1};
-
- const auto& h264_header =
- absl::get<RTPVideoHeaderH264>(video_header.video_type_header);
-
- rtc::CopyOnWriteBuffer result;
- switch (h264_header.packetization_type) {
- case kH264StapA: {
- const uint8_t* payload_end = payload.data() + payload.size();
- const uint8_t* nalu_ptr = payload.data() + 1;
- while (nalu_ptr < payload_end - 1) {
- // The first two bytes describe the length of the segment, where a
- // segment is the nalu type plus nalu payload.
- uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
- nalu_ptr += 2;
-
- if (nalu_ptr + segment_length <= payload_end) {
- result.AppendData(kStartCode);
- result.AppendData(nalu_ptr, segment_length);
- }
- nalu_ptr += segment_length;
- }
- return result;
- }
-
- case kH264FuA: {
- if (IsFirstPacketOfFragment(h264_header)) {
- result.AppendData(kStartCode);
- }
- result.AppendData(payload);
- return result;
- }
-
- case kH264SingleNalu: {
- result.AppendData(kStartCode);
- result.AppendData(payload);
- return result;
- }
- }
-
- RTC_DCHECK_NOTREACHED();
- return result;
-}
-
} // namespace
H26xPacketBuffer::H26xPacketBuffer(bool h264_idr_only_keyframes_allowed)
@@ -162,8 +116,7 @@
packet_slot = std::move(packet);
}
- result.packets = FindFrames(unwrapped_seq_num);
- return result;
+ return FindFrames(unwrapped_seq_num);
}
std::unique_ptr<H26xPacketBuffer::Packet>& H26xPacketBuffer::GetPacket(
@@ -185,9 +138,9 @@
return false;
}
-std::vector<std::unique_ptr<H26xPacketBuffer::Packet>>
-H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
- std::vector<std::unique_ptr<Packet>> found_frames;
+H26xPacketBuffer::InsertResult H26xPacketBuffer::FindFrames(
+ int64_t unwrapped_seq_num) {
+ InsertResult result;
Packet* packet = GetPacket(unwrapped_seq_num).get();
RTC_CHECK(packet != nullptr);
@@ -197,7 +150,7 @@
if (unwrapped_seq_num - 1 != last_continuous_unwrapped_seq_num_) {
if (unwrapped_seq_num <= last_continuous_unwrapped_seq_num_ ||
!BeginningOfStream(*packet)) {
- return found_frames;
+ return result;
}
last_continuous_unwrapped_seq_num_ = unwrapped_seq_num;
@@ -211,7 +164,7 @@
// the 'buffer_'. Check that the `packet` sequence number match the expected
// unwrapped sequence number.
if (static_cast<uint16_t>(seq_num) != packet->seq_num) {
- return found_frames;
+ return result;
}
last_continuous_unwrapped_seq_num_ = seq_num;
@@ -225,12 +178,12 @@
auto& prev_packet = GetPacket(seq_num_start - 1);
if (prev_packet == nullptr || prev_packet->timestamp != rtp_timestamp) {
- if (MaybeAssembleFrame(seq_num_start, seq_num, found_frames)) {
+ if (MaybeAssembleFrame(seq_num_start, seq_num, result)) {
// Frame was assembled, continue to look for more frames.
break;
} else {
// Frame was not assembled, no subsequent frame will be continuous.
- return found_frames;
+ return result;
}
}
}
@@ -239,24 +192,23 @@
seq_num++;
packet = GetPacket(seq_num).get();
if (packet == nullptr) {
- return found_frames;
+ return result;
}
}
- return found_frames;
+ return result;
}
-bool H26xPacketBuffer::MaybeAssembleFrame(
- int64_t start_seq_num_unwrapped,
- int64_t end_sequence_number_unwrapped,
- std::vector<std::unique_ptr<Packet>>& frames) {
+bool H26xPacketBuffer::MaybeAssembleFrame(int64_t start_seq_num_unwrapped,
+ int64_t end_sequence_number_unwrapped,
+ InsertResult& result) {
#ifdef RTC_ENABLE_H265
bool has_vps = false;
#endif
bool has_sps = false;
bool has_pps = false;
+ // Includes IDR, CRA and BLA for HEVC.
bool has_idr = false;
- bool has_irap = false;
int width = -1;
int height = -1;
@@ -284,13 +236,13 @@
for (const auto& nalu_index : nalu_indices) {
uint8_t nalu_type = H265::ParseNaluType(
packet->video_payload.cdata()[nalu_index.payload_start_offset]);
- has_irap |= (nalu_type >= H265::NaluType::kBlaWLp &&
- nalu_type <= H265::NaluType::kRsvIrapVcl23);
+ has_idr |= (nalu_type >= H265::NaluType::kBlaWLp &&
+ nalu_type <= H265::NaluType::kRsvIrapVcl23);
has_vps |= nalu_type == H265::NaluType::kVps;
has_sps |= nalu_type == H265::NaluType::kSps;
has_pps |= nalu_type == H265::NaluType::kPps;
}
- if (has_irap) {
+ if (has_idr) {
if (!has_vps || !has_sps || !has_pps) {
return false;
}
@@ -317,21 +269,253 @@
packet->video_header.height = height;
}
- packet->video_header.frame_type = has_idr || has_irap
+ packet->video_header.frame_type = has_idr
? VideoFrameType::kVideoFrameKey
: VideoFrameType::kVideoFrameDelta;
}
- // Start code is inserted by depacktizer for H.265.
+ // Only applies to H.264 because start code is inserted by depacktizer for
+ // H.265 and out-of-band parameter sets is not supported by H.265.
if (packet->codec() == kVideoCodecH264) {
- packet->video_payload =
- FixH264VideoPayload(packet->video_payload, packet->video_header);
+ if (!FixH264Packet(*packet)) {
+ // The buffer is not cleared actually, but a key frame request is
+ // needed.
+ result.buffer_cleared = true;
+ return false;
+ }
}
- frames.push_back(std::move(packet));
+ result.packets.push_back(std::move(packet));
}
return true;
}
+void H26xPacketBuffer::SetSpropParameterSets(
+ const std::string& sprop_parameter_sets) {
+ if (!h264_idr_only_keyframes_allowed_) {
+ RTC_LOG(LS_WARNING) << "Ignore sprop parameter sets because IDR only "
+ "keyframe is not allowed.";
+ return;
+ }
+ H264SpropParameterSets sprop_decoder;
+ if (!sprop_decoder.DecodeSprop(sprop_parameter_sets)) {
+ return;
+ }
+ InsertSpsPpsNalus(sprop_decoder.sps_nalu(), sprop_decoder.pps_nalu());
+}
+
+void H26xPacketBuffer::InsertSpsPpsNalus(const std::vector<uint8_t>& sps,
+ const std::vector<uint8_t>& pps) {
+ RTC_CHECK(h264_idr_only_keyframes_allowed_);
+ constexpr size_t kNaluHeaderOffset = 1;
+ if (sps.size() < kNaluHeaderOffset) {
+ RTC_LOG(LS_WARNING) << "SPS size " << sps.size() << " is smaller than "
+ << kNaluHeaderOffset;
+ return;
+ }
+ if ((sps[0] & 0x1f) != H264::NaluType::kSps) {
+ RTC_LOG(LS_WARNING) << "SPS Nalu header missing";
+ return;
+ }
+ if (pps.size() < kNaluHeaderOffset) {
+ RTC_LOG(LS_WARNING) << "PPS size " << pps.size() << " is smaller than "
+ << kNaluHeaderOffset;
+ return;
+ }
+ if ((pps[0] & 0x1f) != H264::NaluType::kPps) {
+ RTC_LOG(LS_WARNING) << "SPS Nalu header missing";
+ return;
+ }
+ absl::optional<SpsParser::SpsState> parsed_sps = SpsParser::ParseSps(
+ sps.data() + kNaluHeaderOffset, sps.size() - kNaluHeaderOffset);
+ absl::optional<PpsParser::PpsState> parsed_pps = PpsParser::ParsePps(
+ pps.data() + kNaluHeaderOffset, pps.size() - kNaluHeaderOffset);
+
+ if (!parsed_sps) {
+ RTC_LOG(LS_WARNING) << "Failed to parse SPS.";
+ }
+
+ if (!parsed_pps) {
+ RTC_LOG(LS_WARNING) << "Failed to parse PPS.";
+ }
+
+ if (!parsed_pps || !parsed_sps) {
+ return;
+ }
+
+ SpsInfo sps_info;
+ sps_info.size = sps.size();
+ sps_info.width = parsed_sps->width;
+ sps_info.height = parsed_sps->height;
+ uint8_t* sps_data = new uint8_t[sps_info.size];
+ memcpy(sps_data, sps.data(), sps_info.size);
+ sps_info.payload.reset(sps_data);
+ sps_data_[parsed_sps->id] = std::move(sps_info);
+
+ PpsInfo pps_info;
+ pps_info.size = pps.size();
+ pps_info.sps_id = parsed_pps->sps_id;
+ uint8_t* pps_data = new uint8_t[pps_info.size];
+ memcpy(pps_data, pps.data(), pps_info.size);
+ pps_info.payload.reset(pps_data);
+ pps_data_[parsed_pps->id] = std::move(pps_info);
+
+ RTC_LOG(LS_INFO) << "Inserted SPS id " << parsed_sps->id << " and PPS id "
+ << parsed_pps->id << " (referencing SPS "
+ << parsed_pps->sps_id << ")";
+}
+
+// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to
+// fiddle with the payload at this point.
+bool H26xPacketBuffer::FixH264Packet(Packet& packet) {
+ constexpr uint8_t kStartCode[] = {0, 0, 0, 1};
+
+ RTPVideoHeader& video_header = packet.video_header;
+ RTPVideoHeaderH264& h264_header =
+ absl::get<RTPVideoHeaderH264>(video_header.video_type_header);
+
+ rtc::CopyOnWriteBuffer result;
+
+ if (h264_idr_only_keyframes_allowed_) {
+ // Check if sps and pps insertion is needed.
+ bool prepend_sps_pps = false;
+ auto sps = sps_data_.end();
+ auto pps = pps_data_.end();
+
+ for (size_t i = 0; i < h264_header.nalus_length; ++i) {
+ const NaluInfo& nalu = h264_header.nalus[i];
+ switch (nalu.type) {
+ case H264::NaluType::kSps: {
+ SpsInfo& sps_info = sps_data_[nalu.sps_id];
+ sps_info.width = video_header.width;
+ sps_info.height = video_header.height;
+ break;
+ }
+ case H264::NaluType::kPps: {
+ pps_data_[nalu.pps_id].sps_id = nalu.sps_id;
+ break;
+ }
+ case H264::NaluType::kIdr: {
+ // If this is the first packet of an IDR, make sure we have the
+ // required SPS/PPS and also calculate how much extra space we need
+ // in the buffer to prepend the SPS/PPS to the bitstream with start
+ // codes.
+ if (video_header.is_first_packet_in_frame) {
+ if (nalu.pps_id == -1) {
+ RTC_LOG(LS_WARNING) << "No PPS id in IDR nalu.";
+ return false;
+ }
+
+ pps = pps_data_.find(nalu.pps_id);
+ if (pps == pps_data_.end()) {
+ RTC_LOG(LS_WARNING)
+ << "No PPS with id << " << nalu.pps_id << " received";
+ return false;
+ }
+
+ sps = sps_data_.find(pps->second.sps_id);
+ if (sps == sps_data_.end()) {
+ RTC_LOG(LS_WARNING)
+ << "No SPS with id << " << pps->second.sps_id << " received";
+ return false;
+ }
+
+ // Since the first packet of every keyframe should have its width
+ // and height set we set it here in the case of it being supplied
+ // out of band.
+ video_header.width = sps->second.width;
+ video_header.height = sps->second.height;
+
+ // If the SPS/PPS was supplied out of band then we will have saved
+ // the actual bitstream in `data`.
+ if (sps->second.payload && pps->second.payload) {
+ RTC_DCHECK_GT(sps->second.size, 0);
+ RTC_DCHECK_GT(pps->second.size, 0);
+ prepend_sps_pps = true;
+ }
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ }
+
+ RTC_CHECK(!prepend_sps_pps ||
+ (sps != sps_data_.end() && pps != pps_data_.end()));
+
+ // Insert SPS and PPS if they are missing.
+ if (prepend_sps_pps) {
+ // Insert SPS.
+ result.AppendData(kStartCode);
+ result.AppendData(sps->second.payload.get(), sps->second.size);
+
+ // Insert PPS.
+ result.AppendData(kStartCode);
+ result.AppendData(pps->second.payload.get(), pps->second.size);
+
+ // Update codec header to reflect the newly added SPS and PPS.
+ NaluInfo sps_info;
+ sps_info.type = H264::NaluType::kSps;
+ sps_info.sps_id = sps->first;
+ sps_info.pps_id = -1;
+ NaluInfo pps_info;
+ pps_info.type = H264::NaluType::kPps;
+ pps_info.sps_id = sps->first;
+ pps_info.pps_id = pps->first;
+ if (h264_header.nalus_length + 2 <= kMaxNalusPerPacket) {
+ h264_header.nalus[h264_header.nalus_length++] = sps_info;
+ h264_header.nalus[h264_header.nalus_length++] = pps_info;
+ } else {
+ RTC_LOG(LS_WARNING)
+ << "Not enough space in H.264 codec header to insert "
+ "SPS/PPS provided out-of-band.";
+ }
+ }
+ }
+
+ // Insert start code.
+ switch (h264_header.packetization_type) {
+ case kH264StapA: {
+ const uint8_t* payload_end =
+ packet.video_payload.data() + packet.video_payload.size();
+ const uint8_t* nalu_ptr = packet.video_payload.data() + 1;
+ while (nalu_ptr < payload_end - 1) {
+ // The first two bytes describe the length of the segment, where a
+ // segment is the nalu type plus nalu payload.
+ uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
+ nalu_ptr += 2;
+
+ if (nalu_ptr + segment_length <= payload_end) {
+ result.AppendData(kStartCode);
+ result.AppendData(nalu_ptr, segment_length);
+ }
+ nalu_ptr += segment_length;
+ }
+ packet.video_payload = result;
+ return true;
+ }
+
+ case kH264FuA: {
+ if (IsFirstPacketOfFragment(h264_header)) {
+ result.AppendData(kStartCode);
+ }
+ result.AppendData(packet.video_payload);
+ packet.video_payload = result;
+ return true;
+ }
+
+ case kH264SingleNalu: {
+ result.AppendData(kStartCode);
+ result.AppendData(packet.video_payload);
+ packet.video_payload = result;
+ return true;
+ }
+ }
+
+ RTC_DCHECK_NOTREACHED();
+ return false;
+}
+
} // namespace webrtc
diff --git a/modules/video_coding/h26x_packet_buffer.h b/modules/video_coding/h26x_packet_buffer.h
index 2160156..8bfae71 100644
--- a/modules/video_coding/h26x_packet_buffer.h
+++ b/modules/video_coding/h26x_packet_buffer.h
@@ -12,7 +12,9 @@
#define MODULES_VIDEO_CODING_H26X_PACKET_BUFFER_H_
#include <array>
+#include <map>
#include <memory>
+#include <string>
#include <vector>
#include "absl/base/attributes.h"
@@ -36,20 +38,68 @@
ABSL_MUST_USE_RESULT InsertResult
InsertPacket(std::unique_ptr<Packet> packet);
+ // Out of band supplied codec parameters for H.264.
+ void SetSpropParameterSets(const std::string& sprop_parameter_sets);
+
private:
+ // Stores PPS payload and the active SPS ID.
+ struct PpsInfo {
+ PpsInfo() = default;
+ PpsInfo(PpsInfo&& rhs) = default;
+ PpsInfo& operator=(PpsInfo&& rhs) = default;
+ ~PpsInfo() = default;
+
+ // The value of sps_seq_parameter_set_id for the active SPS.
+ uint32_t sps_id = 0;
+ // Payload size.
+ size_t size = 0;
+ std::unique_ptr<uint8_t[]> payload;
+ };
+
+ // Stores SPS payload and picture size.
+ struct SpsInfo {
+ SpsInfo() = default;
+ SpsInfo(SpsInfo&& rhs) = default;
+ SpsInfo& operator=(SpsInfo&& rhs) = default;
+ ~SpsInfo() = default;
+
+ // The width and height of decoded pictures.
+ int width = -1;
+ int height = -1;
+ // Payload size.
+ size_t size = 0;
+ std::unique_ptr<uint8_t[]> payload;
+ };
+
static constexpr int kBufferSize = 2048;
std::unique_ptr<Packet>& GetPacket(int64_t unwrapped_seq_num);
bool BeginningOfStream(const Packet& packet) const;
- std::vector<std::unique_ptr<Packet>> FindFrames(int64_t unwrapped_seq_num);
+ InsertResult FindFrames(int64_t unwrapped_seq_num);
bool MaybeAssembleFrame(int64_t start_seq_num_unwrapped,
int64_t end_sequence_number_unwrapped,
- std::vector<std::unique_ptr<Packet>>& packets);
+ InsertResult& result);
+ // Store SPS and PPS nalus. They will be used later when an IDR frame is
+ // received without SPS/PPS.
+ void InsertSpsPpsNalus(const std::vector<uint8_t>& sps,
+ const std::vector<uint8_t>& pps);
+ // Insert start code and paramter sets for H.264 payload, also update header
+ // if parameter sets are inserted. Return false if required SPS or PPS is not
+ // found.
+ bool FixH264Packet(Packet& packet);
+ // Indicates whether IDR frames without SPS and PPS are allowed.
const bool h264_idr_only_keyframes_allowed_;
std::array<std::unique_ptr<Packet>, kBufferSize> buffer_;
absl::optional<int64_t> last_continuous_unwrapped_seq_num_;
SeqNumUnwrapper<uint16_t> seq_num_unwrapper_;
+
+ // Map from pps_pic_parameter_set_id to the PPS payload associated with this
+ // ID.
+ std::map<uint32_t, PpsInfo> pps_data_;
+ // Map from sps_video_parameter_set_id to the SPS payload associated with this
+ // ID.
+ std::map<uint32_t, SpsInfo> sps_data_;
};
} // namespace webrtc
diff --git a/modules/video_coding/h26x_packet_buffer_unittest.cc b/modules/video_coding/h26x_packet_buffer_unittest.cc
index ac5bcb7..8d6d691 100644
--- a/modules/video_coding/h26x_packet_buffer_unittest.cc
+++ b/modules/video_coding/h26x_packet_buffer_unittest.cc
@@ -42,6 +42,11 @@
using H264::NaluType::kStapA;
constexpr int kBufferSize = 2048;
+// Example sprop string from https://tools.ietf.org/html/rfc3984.
+const char kExampleSpropString[] = "Z0IACpZTBYmI,aMljiA==";
+static const std::vector<uint8_t> kExampleSpropRawSps{
+ 0x67, 0x42, 0x00, 0x0A, 0x96, 0x53, 0x05, 0x89, 0x88};
+static const std::vector<uint8_t> kExampleSpropRawPps{0x68, 0xC9, 0x63, 0x88};
std::vector<uint8_t> StartCode() {
return {0, 0, 0, 1};
@@ -59,12 +64,14 @@
public:
explicit H264Packet(H264PacketizationTypes type);
- H264Packet& Idr(std::vector<uint8_t> payload = {9, 9, 9});
+ H264Packet& Idr(std::vector<uint8_t> payload = {9, 9, 9}, int pps_id = -1);
H264Packet& Slice(std::vector<uint8_t> payload = {9, 9, 9});
- H264Packet& Sps(std::vector<uint8_t> payload = {9, 9, 9});
+ H264Packet& Sps(std::vector<uint8_t> payload = {9, 9, 9}, int sps_id = -1);
H264Packet& SpsWithResolution(RenderResolution resolution,
std::vector<uint8_t> payload = {9, 9, 9});
- H264Packet& Pps(std::vector<uint8_t> payload = {9, 9, 9});
+ H264Packet& Pps(std::vector<uint8_t> payload = {9, 9, 9},
+ int pps_id = -1,
+ int sps_id = -1);
H264Packet& Aud();
H264Packet& Marker();
H264Packet& AsFirstFragment();
@@ -98,9 +105,11 @@
video_header_.video_type_header.emplace<RTPVideoHeaderH264>();
}
-H264Packet& H264Packet::Idr(std::vector<uint8_t> payload) {
+H264Packet& H264Packet::Idr(std::vector<uint8_t> payload, int pps_id) {
auto& h264_header = H264Header();
- h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kIdr);
+ auto nalu_info = MakeNaluInfo(kIdr);
+ nalu_info.pps_id = pps_id;
+ h264_header.nalus[h264_header.nalus_length++] = nalu_info;
nalu_payloads_.push_back(std::move(payload));
return *this;
}
@@ -112,9 +121,11 @@
return *this;
}
-H264Packet& H264Packet::Sps(std::vector<uint8_t> payload) {
+H264Packet& H264Packet::Sps(std::vector<uint8_t> payload, int sps_id) {
auto& h264_header = H264Header();
- h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps);
+ auto nalu_info = MakeNaluInfo(kSps);
+ nalu_info.pps_id = sps_id;
+ h264_header.nalus[h264_header.nalus_length++] = nalu_info;
nalu_payloads_.push_back(std::move(payload));
return *this;
}
@@ -129,9 +140,14 @@
return *this;
}
-H264Packet& H264Packet::Pps(std::vector<uint8_t> payload) {
+H264Packet& H264Packet::Pps(std::vector<uint8_t> payload,
+ int pps_id,
+ int sps_id) {
auto& h264_header = H264Header();
- h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kPps);
+ auto nalu_info = MakeNaluInfo(kPps);
+ nalu_info.pps_id = pps_id;
+ nalu_info.sps_id = sps_id;
+ h264_header.nalus[h264_header.nalus_length++] = nalu_info;
nalu_payloads_.push_back(std::move(payload));
return *this;
}
@@ -354,14 +370,97 @@
return res;
}
-TEST(H26xPacketBufferTest, IdrIsKeyframe) {
+TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithSprop) {
+ H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+ packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+ auto packets =
+ packet_buffer
+ .InsertPacket(
+ H264Packet(kH264SingleNalu).Idr({1, 2, 3}, 0).Marker().Build())
+ .packets;
+ EXPECT_THAT(packets, SizeIs(1));
+ EXPECT_THAT(PacketPayload(packets[0]),
+ ElementsAreArray(FlatVector({StartCode(),
+ kExampleSpropRawSps,
+ StartCode(),
+ kExampleSpropRawPps,
+ StartCode(),
+ {kIdr, 1, 2, 3}})));
+}
+
+TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithoutSprop) {
H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
- EXPECT_THAT(
+ // Cannot fix biststream by prepending SPS and PPS because no sprop string is
+ // available. Request a key frame.
+ EXPECT_TRUE(
packet_buffer
- .InsertPacket(H264Packet(kH264SingleNalu).Idr().Marker().Build())
- .packets,
- SizeIs(1));
+ .InsertPacket(
+ H264Packet(kH264SingleNalu).Idr({9, 9, 9}, 0).Marker().Build())
+ .buffer_cleared);
+}
+
+TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithSpropAndUnknownPpsId) {
+ H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+ packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+ // Cannot fix biststream because sprop string doesn't contain a PPS with given
+ // ID. Request a key frame.
+ EXPECT_TRUE(
+ packet_buffer
+ .InsertPacket(
+ H264Packet(kH264SingleNalu).Idr({9, 9, 9}, 1).Marker().Build())
+ .buffer_cleared);
+}
+
+TEST(H26xPacketBufferTest, IdrOnlyKeyframeInTheMiddle) {
+ H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+ packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+ RTC_UNUSED(packet_buffer.InsertPacket(
+ H264Packet(kH264SingleNalu).Sps({1, 2, 3}, 1).SeqNum(0).Time(0).Build()));
+ RTC_UNUSED(packet_buffer.InsertPacket(H264Packet(kH264SingleNalu)
+ .Pps({4, 5, 6}, 1, 1)
+ .SeqNum(1)
+ .Time(0)
+ .Build()));
+ EXPECT_THAT(packet_buffer
+ .InsertPacket(H264Packet(kH264SingleNalu)
+ .Idr({7, 8, 9}, 1)
+ .SeqNum(2)
+ .Time(0)
+ .Marker()
+ .Build())
+ .packets,
+ SizeIs(3));
+
+ EXPECT_THAT(packet_buffer
+ .InsertPacket(H264Packet(kH264SingleNalu)
+ .Slice()
+ .SeqNum(3)
+ .Time(1)
+ .Marker()
+ .Build())
+ .packets,
+ SizeIs(1));
+
+ auto packets = packet_buffer
+ .InsertPacket(H264Packet(kH264SingleNalu)
+ .Idr({10, 11, 12}, 0)
+ .SeqNum(4)
+ .Time(2)
+ .Marker()
+ .Build())
+ .packets;
+ EXPECT_THAT(packets, SizeIs(1));
+ EXPECT_THAT(PacketPayload(packets[0]),
+ ElementsAreArray(FlatVector({StartCode(),
+ kExampleSpropRawSps,
+ StartCode(),
+ kExampleSpropRawPps,
+ StartCode(),
+ {kIdr, 10, 11, 12}})));
}
TEST(H26xPacketBufferTest, IdrIsNotKeyframe) {
@@ -376,6 +475,7 @@
TEST(H26xPacketBufferTest, IdrIsKeyframeFuaRequiresFirstFragmet) {
H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+ packet_buffer.SetSpropParameterSets(kExampleSpropString);
// Not marked as the first fragment
EXPECT_THAT(
@@ -394,7 +494,7 @@
// Marked as first fragment
EXPECT_THAT(packet_buffer
.InsertPacket(H264Packet(kH264FuA)
- .Idr()
+ .Idr({9, 9, 9}, 0)
.SeqNum(2)
.Time(1)
.AsFirstFragment()
@@ -428,6 +528,37 @@
SizeIs(3));
}
+TEST(H26xPacketBufferTest, SpsPpsIdrIsKeyframeIgnoresSprop) {
+ H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false);
+
+ // When h264_allow_idr_only_keyframes is false, sprop string should be
+ // ignored. Use in band parameter sets.
+ packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+ RTC_UNUSED(packet_buffer.InsertPacket(
+ H264Packet(kH264SingleNalu).Sps({1, 2, 3}, 0).SeqNum(0).Time(0).Build()));
+ RTC_UNUSED(packet_buffer.InsertPacket(H264Packet(kH264SingleNalu)
+ .Pps({4, 5, 6}, 0, 0)
+ .SeqNum(1)
+ .Time(0)
+ .Build()));
+ auto packets = packet_buffer
+ .InsertPacket(H264Packet(kH264SingleNalu)
+ .Idr({7, 8, 9}, 0)
+ .SeqNum(2)
+ .Time(0)
+ .Marker()
+ .Build())
+ .packets;
+ EXPECT_THAT(packets, SizeIs(3));
+ EXPECT_THAT(PacketPayload(packets[0]),
+ ElementsAreArray(FlatVector({StartCode(), {kSps, 1, 2, 3}})));
+ EXPECT_THAT(PacketPayload(packets[1]),
+ ElementsAreArray(FlatVector({StartCode(), {kPps, 4, 5, 6}})));
+ EXPECT_THAT(PacketPayload(packets[2]),
+ ElementsAreArray(FlatVector({StartCode(), {kIdr, 7, 8, 9}})));
+}
+
TEST(H26xPacketBufferTest, PpsIdrIsNotKeyframeSingleNalus) {
H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false);
@@ -966,6 +1097,15 @@
IsEmpty());
}
+TEST(H26xPacketBufferTest, H265IdrIsNotKeyFrameEvenWithSprop) {
+ H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+ packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+ EXPECT_THAT(
+ packet_buffer.InsertPacket(H265Packet().Idr().Marker().Build()).packets,
+ IsEmpty());
+}
+
TEST(H26xPacketBufferTest, H265SpsPpsIdrIsNotKeyFrame) {
H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false);
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 8752929..9f20746 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -128,6 +128,7 @@
"../modules/rtp_rtcp:rtp_rtcp_format",
"../modules/rtp_rtcp:rtp_video_header",
"../modules/video_coding",
+ "../modules/video_coding:h264_sprop_parameter_sets",
"../modules/video_coding:nack_requester",
"../modules/video_coding:packet_buffer",
"../modules/video_coding:video_codec_interface",
