Adding test for SingleNalUnit mode
Test enables single-nalu mode, sets limit for nalu lenght and verifies
that encoder follows that limit.
I found that QP jumps significantly when the mode is enabled. In result
encoder might produce 4kbyte and 0.4kbyte frames back-to-back. But it
seems that happens only to couple of frames in the beginning. This
caused test to fail with default RC thresholds. To bypass this I
increased frame size mismatch threshold from 20 to 30%. This should be
Ok considering single-nalu mode is rare.
BUG=webrtc:8070
Review-Url: https://codereview.webrtc.org/3014623002
Cr-Commit-Position: refs/heads/master@{#20023}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 2371d08..785533a 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -377,6 +377,7 @@
}
deps = [
+ ":codec_globals_headers",
":video_coding",
":video_coding_utility",
":webrtc_vp8",
diff --git a/modules/video_coding/codecs/test/stats.h b/modules/video_coding/codecs/test/stats.h
index 8022f27..fd6fa87 100644
--- a/modules/video_coding/codecs/test/stats.h
+++ b/modules/video_coding/codecs/test/stats.h
@@ -32,6 +32,9 @@
size_t encoded_frame_size_bytes = 0;
webrtc::FrameType frame_type = kVideoFrameDelta;
+ // H264 specific.
+ rtc::Optional<size_t> max_nalu_length;
+
// Decoding.
int64_t decode_start_ns = 0;
int decode_return_code = 0;
diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc
index af5b4b3..cc453fa 100644
--- a/modules/video_coding/codecs/test/videoprocessor.cc
+++ b/modules/video_coding/codecs/test/videoprocessor.cc
@@ -12,6 +12,7 @@
#include <string.h>
+#include <algorithm>
#include <limits>
#include <memory>
#include <utility>
@@ -19,6 +20,7 @@
#include "api/video/i420_buffer.h"
#include "common_types.h" // NOLINT(build/include)
+#include "common_video/h264/h264_common.h"
#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
#include "modules/video_coding/include/video_codec_initializer.h"
#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
@@ -108,6 +110,24 @@
EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP.";
}
+rtc::Optional<size_t> GetMaxNaluLength(const EncodedImage& encoded_frame,
+ const TestConfig& config) {
+ if (config.codec_settings.codecType != kVideoCodecH264)
+ return rtc::Optional<size_t>();
+
+ std::vector<webrtc::H264::NaluIndex> nalu_indices =
+ webrtc::H264::FindNaluIndices(encoded_frame._buffer,
+ encoded_frame._length);
+
+ RTC_CHECK(!nalu_indices.empty());
+
+ size_t max_length = 0;
+ for (const webrtc::H264::NaluIndex& index : nalu_indices)
+ max_length = std::max(max_length, index.payload_size);
+
+ return rtc::Optional<size_t>(max_length);
+}
+
int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) {
int64_t diff_us = (stop_ns - start_ns) / rtc::kNumNanosecsPerMicrosec;
RTC_DCHECK_GE(diff_us, std::numeric_limits<int>::min());
@@ -351,6 +371,8 @@
encoded_image._length / config_.networking_config.packet_size_in_bytes +
1;
+ frame_stat->max_nalu_length = GetMaxNaluLength(encoded_image, config_);
+
// Simulate packet loss.
bool exclude_this_frame = false;
if (encoded_image._frameType == kVideoFrameKey) {
diff --git a/modules/video_coding/codecs/test/videoprocessor.h b/modules/video_coding/codecs/test/videoprocessor.h
index 98b9140..cf96923 100644
--- a/modules/video_coding/codecs/test/videoprocessor.h
+++ b/modules/video_coding/codecs/test/videoprocessor.h
@@ -18,6 +18,7 @@
#include "api/video/video_frame.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "modules/video_coding/codecs/h264/include/h264_globals.h"
#include "modules/video_coding/codecs/test/packet_manipulator.h"
#include "modules/video_coding/codecs/test/stats.h"
#include "modules/video_coding/include/video_codec_interface.h"
@@ -100,6 +101,10 @@
// Should the hardware codecs be wrapped in software fallbacks?
bool sw_fallback_encoder = false;
bool sw_fallback_decoder = false;
+
+ // RTP H264 packetization mode.
+ H264PacketizationMode packetization_mode =
+ H264PacketizationMode::NonInterleaved;
};
// Handles encoding/decoding of video using the VideoEncoder/VideoDecoder
diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
index 99ecb10..7d9a81a 100644
--- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
@@ -176,6 +176,7 @@
const RateProfile& rate_profile,
const std::vector<RateControlThresholds>* rc_thresholds,
const QualityThresholds* quality_thresholds,
+ const BitstreamThresholds* bs_thresholds,
const VisualizationParams* visualization_params) {
// The Android HW codec needs to be run on a task queue, so we simply always
// run the test on a task queue.
@@ -246,6 +247,10 @@
while (frame_number < num_frames) {
UpdateRateControlMetrics(frame_number);
+ if (bs_thresholds) {
+ VerifyBitstream(frame_number, *bs_thresholds);
+ }
+
++frame_number;
if (frame_number ==
@@ -336,6 +341,13 @@
case kVideoCodecH264:
// TODO(brandtr): Generalize so that we support multiple profiles here.
codec = cricket::VideoCodec(cricket::kH264CodecName);
+ if (config_.packetization_mode == H264PacketizationMode::NonInterleaved) {
+ codec.SetParam(cricket::kH264FmtpPacketizationMode, "1");
+ } else {
+ RTC_CHECK_EQ(config_.packetization_mode,
+ H264PacketizationMode::SingleNalUnit);
+ codec.SetParam(cricket::kH264FmtpPacketizationMode, "0");
+ }
encoder_.reset(encoder_factory->CreateVideoEncoder(codec));
decoder_.reset(
decoder_factory->CreateVideoDecoderWithParams(codec, decoder_params));
@@ -562,6 +574,14 @@
printf("\n");
}
+void VideoProcessorIntegrationTest::VerifyBitstream(
+ int frame_number,
+ const BitstreamThresholds& bs_thresholds) {
+ RTC_CHECK_GE(frame_number, 0);
+ const FrameStatistic* frame_stat = stats_.GetFrame(frame_number);
+ EXPECT_LE(*(frame_stat->max_nalu_length), bs_thresholds.max_nalu_length);
+}
+
// Temporal layer index corresponding to frame number, for up to 3 layers.
int VideoProcessorIntegrationTest::TemporalLayerIndexForFrame(
int frame_number) const {
diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
index 9831e48..d383915 100644
--- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
@@ -71,6 +71,12 @@
double min_min_ssim;
};
+struct BitstreamThresholds {
+ explicit BitstreamThresholds(size_t max_nalu_length)
+ : max_nalu_length(max_nalu_length) {}
+ size_t max_nalu_length;
+};
+
// Should video files be saved persistently to disk for post-run visualization?
struct VisualizationParams {
bool save_encoded_ivf;
@@ -122,6 +128,7 @@
const RateProfile& rate_profile,
const std::vector<RateControlThresholds>* rc_thresholds,
const QualityThresholds* quality_thresholds,
+ const BitstreamThresholds* bs_thresholds,
const VisualizationParams* visualization_params);
// Config.
@@ -192,6 +199,9 @@
const std::vector<int>& num_dropped_frames,
const std::vector<int>& num_spatial_resizes) const;
+ void VerifyBitstream(int frame_number,
+ const BitstreamThresholds& bs_thresholds);
+
// Codecs.
std::unique_ptr<VideoEncoder> encoder_;
std::unique_ptr<VideoDecoder> decoder_;
diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc
index 9f3c1d3..3e43a76 100644
--- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc
@@ -70,7 +70,7 @@
QualityThresholds quality_thresholds(37.0, 36.0, 0.93, 0.92);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// VP9: Run with 5% packet loss and fixed bitrate. Quality should be a bit
@@ -91,7 +91,7 @@
QualityThresholds quality_thresholds(17.0, 14.0, 0.45, 0.36);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// VP9: Run with no packet loss, with varying bitrate (3 rate updates):
@@ -117,7 +117,7 @@
QualityThresholds quality_thresholds(35.5, 30.0, 0.90, 0.85);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// VP9: Run with no packet loss, with an update (decrease) in frame rate.
@@ -147,7 +147,7 @@
QualityThresholds quality_thresholds(31.5, 18.0, 0.80, 0.43);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// VP9: Run with no packet loss and denoiser on. One key frame (first frame).
@@ -166,7 +166,7 @@
QualityThresholds quality_thresholds(36.8, 35.8, 0.92, 0.91);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// Run with no packet loss, at low bitrate.
@@ -188,7 +188,7 @@
QualityThresholds quality_thresholds(24.0, 13.0, 0.65, 0.37);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// TODO(marpan): Add temporal layer test for VP9, once changes are in
@@ -214,7 +214,7 @@
QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// VP8: Run with 5% packet loss and fixed bitrate. Quality should be a bit
@@ -235,7 +235,7 @@
QualityThresholds quality_thresholds(20.0, 16.0, 0.60, 0.40);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// VP8: Run with 10% packet loss and fixed bitrate. Quality should be lower.
@@ -256,7 +256,7 @@
QualityThresholds quality_thresholds(19.0, 16.0, 0.50, 0.35);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
#endif // !defined(WEBRTC_IOS)
@@ -301,7 +301,7 @@
QualityThresholds quality_thresholds(34.0, 32.0, 0.85, 0.80);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// VP8: Run with no packet loss, with an update (decrease) in frame rate.
@@ -339,7 +339,7 @@
QualityThresholds quality_thresholds(31.0, 22.0, 0.80, 0.65);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
// VP8: Run with no packet loss, with 3 temporal layers, with a rate update in
@@ -372,7 +372,7 @@
QualityThresholds quality_thresholds(32.5, 30.0, 0.85, 0.80);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
} // namespace test
diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc
index 293ac54..6e20244 100644
--- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc
@@ -57,7 +57,7 @@
QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
TEST_F(VideoProcessorIntegrationTestMediaCodec,
@@ -89,7 +89,7 @@
QualityThresholds quality_thresholds(33.0, 30.0, 0.90, 0.85);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
}
#endif // defined(WEBRTC_ANDROID)
diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc
index e6db027..df3e4d0 100644
--- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc
@@ -53,7 +53,7 @@
// these unittests appears to drop "packets" in a way that is not compatible
// with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have
// not been added.
-TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLossH264) {
+TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLoss) {
SetCodecSettings(&config_, kVideoCodecH264, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
@@ -68,7 +68,32 @@
QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
- kNoVisualizationParams);
+ nullptr, kNoVisualizationParams);
+}
+
+// H264: Enable SingleNalUnit packetization mode. Encoder should split
+// large frames into multiple slices and limit length of NAL units.
+TEST_F(VideoProcessorIntegrationTestOpenH264, ProcessNoLossSingleNalUnit) {
+ config_.packetization_mode = H264PacketizationMode::SingleNalUnit;
+ config_.networking_config.max_payload_size_in_bytes = 500;
+ SetCodecSettings(&config_, kVideoCodecH264, 1, false, false, true, false,
+ kResilienceOn, kCifWidth, kCifHeight);
+
+ RateProfile rate_profile;
+ SetRateProfile(&rate_profile, 0, 500, 30, 0);
+ rate_profile.frame_index_rate_update[1] = kNumFrames + 1;
+ rate_profile.num_frames = kNumFrames;
+
+ std::vector<RateControlThresholds> rc_thresholds;
+ AddRateControlThresholds(2, 60, 30, 10, 20, 0, 1, &rc_thresholds);
+
+ QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70);
+
+ BitstreamThresholds bs_thresholds(
+ config_.networking_config.max_payload_size_in_bytes);
+
+ ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
+ &bs_thresholds, kNoVisualizationParams);
}
#endif // defined(WEBRTC_USE_H264)
diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc
index 4bd5fec..8a182f9 100644
--- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc
@@ -77,7 +77,7 @@
rate_profile.frame_index_rate_update[1] = kNumFrames + 1;
rate_profile.num_frames = kNumFrames;
- ProcessFramesAndMaybeVerify(rate_profile, nullptr, nullptr,
+ ProcessFramesAndMaybeVerify(rate_profile, nullptr, nullptr, nullptr,
&kVisualizationParams);
}
