Parse encoded frame QP if not provided by encoder
Bug: webrtc:12542
Change-Id: Ic70b46e226f158db7a478a9f20e1f940804febba
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/210966
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33434}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 2a50436..8bb9070 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -323,6 +323,8 @@
"utility/ivf_file_reader.h",
"utility/ivf_file_writer.cc",
"utility/ivf_file_writer.h",
+ "utility/qp_parser.cc",
+ "utility/qp_parser.h",
"utility/quality_scaler.cc",
"utility/quality_scaler.h",
"utility/simulcast_rate_allocator.cc",
@@ -969,6 +971,7 @@
"utility/framerate_controller_unittest.cc",
"utility/ivf_file_reader_unittest.cc",
"utility/ivf_file_writer_unittest.cc",
+ "utility/qp_parser_unittest.cc",
"utility/quality_scaler_unittest.cc",
"utility/simulcast_rate_allocator_unittest.cc",
"video_codec_initializer_unittest.cc",
diff --git a/modules/video_coding/utility/qp_parser.cc b/modules/video_coding/utility/qp_parser.cc
new file mode 100644
index 0000000..71958d0
--- /dev/null
+++ b/modules/video_coding/utility/qp_parser.cc
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/qp_parser.h"
+
+#include <utility>
+
+#include "modules/video_coding/utility/vp8_header_parser.h"
+#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
+
+namespace webrtc {
+
+absl::optional<uint32_t> QpParser::Parse(VideoCodecType codec_type,
+ size_t spatial_idx,
+ const uint8_t* frame_data,
+ size_t frame_size) {
+ if (frame_data == nullptr || frame_size == 0) {
+ return absl::nullopt;
+ }
+
+ if (codec_type == kVideoCodecVP8) {
+ int qp = -1;
+ if (vp8::GetQp(frame_data, frame_size, &qp)) {
+ return qp;
+ }
+ } else if (codec_type == kVideoCodecVP9) {
+ int qp = -1;
+ if (vp9::GetQp(frame_data, frame_size, &qp)) {
+ return qp;
+ }
+ } else if (codec_type == kVideoCodecH264) {
+ H264BitstreamParser& parser = FetchOrCreateH264Parser(spatial_idx);
+ parser.ParseBitstream(
+ rtc::ArrayView<const uint8_t>(frame_data, frame_size));
+ return parser.GetLastSliceQp();
+ }
+
+ return absl::nullopt;
+}
+
+H264BitstreamParser& QpParser::FetchOrCreateH264Parser(size_t spatial_idx) {
+ if (h264_parsers_.find(spatial_idx) == h264_parsers_.end()) {
+ h264_parsers_.emplace(std::make_pair(spatial_idx, H264BitstreamParser()));
+ }
+ return h264_parsers_.at(spatial_idx);
+}
+
+} // namespace webrtc
diff --git a/modules/video_coding/utility/qp_parser.h b/modules/video_coding/utility/qp_parser.h
new file mode 100644
index 0000000..5175e37
--- /dev/null
+++ b/modules/video_coding/utility/qp_parser.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_
+#define MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_
+
+#include <map>
+
+#include "absl/types/optional.h"
+#include "api/video/video_codec_type.h"
+#include "common_video/h264/h264_bitstream_parser.h"
+
+namespace webrtc {
+
+class QpParser {
+ public:
+ absl::optional<uint32_t> Parse(VideoCodecType codec_type,
+ size_t spatial_idx,
+ const uint8_t* frame_data,
+ size_t frame_size);
+
+ private:
+ H264BitstreamParser& FetchOrCreateH264Parser(size_t spatial_idx);
+
+ std::map<size_t, H264BitstreamParser> h264_parsers_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_
diff --git a/modules/video_coding/utility/qp_parser_unittest.cc b/modules/video_coding/utility/qp_parser_unittest.cc
new file mode 100644
index 0000000..1eff2e8
--- /dev/null
+++ b/modules/video_coding/utility/qp_parser_unittest.cc
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/qp_parser.h"
+
+#include <stddef.h>
+
+#include "test/gtest.h"
+
+namespace webrtc {
+
+namespace {
+// ffmpeg -s 16x16 -f rawvideo -pix_fmt rgb24 -r 30 -i /dev/zero -c:v libvpx
+// -qmin 20 -qmax 20 -crf 20 -frames:v 1 -y out.ivf
+const uint8_t kCodedFrameVp8Qp25[] = {
+ 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
+ 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
+ 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
+
+// ffmpeg -s 16x16 -f rawvideo -pix_fmt rgb24 -r 30 -i /dev/zero -c:v libvpx-vp9
+// -qmin 24 -qmax 24 -crf 24 -frames:v 1 -y out.ivf
+const uint8_t kCodedFrameVp9Qp96[] = {
+ 0xa2, 0x49, 0x83, 0x42, 0xe0, 0x00, 0xf0, 0x00, 0xf6, 0x00,
+ 0x38, 0x24, 0x1c, 0x18, 0xc0, 0x00, 0x00, 0x30, 0x70, 0x00,
+ 0x00, 0x4a, 0xa7, 0xff, 0xfc, 0xb9, 0x01, 0xbf, 0xff, 0xff,
+ 0x97, 0x20, 0xdb, 0xff, 0xff, 0xcb, 0x90, 0x5d, 0x40};
+
+// ffmpeg -s 16x16 -f rawvideo -pix_fmt yuv420p -r 30 -i /dev/zero -c:v libx264
+// -qmin 38 -qmax 38 -crf 38 -profile:v baseline -frames:v 2 -y out.264
+const uint8_t kCodedFrameH264SpsPpsIdrQp38[] = {
+ 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x0a, 0xd9, 0x1e, 0x84,
+ 0x00, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c,
+ 0x48, 0x99, 0x20, 0x00, 0x00, 0x00, 0x01, 0x68, 0xcb, 0x80, 0xc4,
+ 0xb2, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0xf1, 0x18, 0xa0, 0x00,
+ 0x20, 0x5b, 0x1c, 0x00, 0x04, 0x07, 0xe3, 0x80, 0x00, 0x80, 0xfe};
+
+const uint8_t kCodedFrameH264SpsPpsIdrQp49[] = {
+ 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x0a, 0xd9, 0x1e, 0x84,
+ 0x00, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c,
+ 0x48, 0x99, 0x20, 0x00, 0x00, 0x00, 0x01, 0x68, 0xcb, 0x80, 0x5d,
+ 0x2c, 0x80, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0xf1, 0x18, 0xa0,
+ 0x00, 0x5e, 0x38, 0x00, 0x08, 0x03, 0xc7, 0x00, 0x01, 0x00, 0x7c};
+
+const uint8_t kCodedFrameH264InterSliceQpDelta0[] = {0x00, 0x00, 0x00, 0x01,
+ 0x41, 0x9a, 0x39, 0xea};
+
+} // namespace
+
+TEST(QpParserTest, ParseQpVp8) {
+ QpParser parser;
+ absl::optional<uint32_t> qp = parser.Parse(
+ kVideoCodecVP8, 0, kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25));
+ EXPECT_EQ(qp, 25u);
+}
+
+TEST(QpParserTest, ParseQpVp9) {
+ QpParser parser;
+ absl::optional<uint32_t> qp = parser.Parse(
+ kVideoCodecVP9, 0, kCodedFrameVp9Qp96, sizeof(kCodedFrameVp9Qp96));
+ EXPECT_EQ(qp, 96u);
+}
+
+TEST(QpParserTest, ParseQpH264) {
+ QpParser parser;
+ absl::optional<uint32_t> qp = parser.Parse(
+ VideoCodecType::kVideoCodecH264, 0, kCodedFrameH264SpsPpsIdrQp38,
+ sizeof(kCodedFrameH264SpsPpsIdrQp38));
+ EXPECT_EQ(qp, 38u);
+
+ qp = parser.Parse(kVideoCodecH264, 1, kCodedFrameH264SpsPpsIdrQp49,
+ sizeof(kCodedFrameH264SpsPpsIdrQp49));
+ EXPECT_EQ(qp, 49u);
+
+ qp = parser.Parse(kVideoCodecH264, 0, kCodedFrameH264InterSliceQpDelta0,
+ sizeof(kCodedFrameH264InterSliceQpDelta0));
+ EXPECT_EQ(qp, 38u);
+
+ qp = parser.Parse(kVideoCodecH264, 1, kCodedFrameH264InterSliceQpDelta0,
+ sizeof(kCodedFrameH264InterSliceQpDelta0));
+ EXPECT_EQ(qp, 49u);
+}
+
+TEST(QpParserTest, ParseQpUnsupportedCodecType) {
+ QpParser parser;
+ absl::optional<uint32_t> qp = parser.Parse(
+ kVideoCodecGeneric, 0, kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25));
+ EXPECT_FALSE(qp.has_value());
+}
+
+TEST(QpParserTest, ParseQpNullData) {
+ QpParser parser;
+ absl::optional<uint32_t> qp =
+ parser.Parse(kVideoCodecGeneric, 0, nullptr, 100);
+ EXPECT_FALSE(qp.has_value());
+}
+
+TEST(QpParserTest, ParseQpEmptyData) {
+ QpParser parser;
+ absl::optional<uint32_t> qp =
+ parser.Parse(kVideoCodecGeneric, 0, kCodedFrameVp8Qp25, 0);
+ EXPECT_FALSE(qp.has_value());
+}
+
+} // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index cfef876..db61e7a 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -658,6 +658,8 @@
/*source=*/nullptr),
default_limits_allowed_(
!field_trial::IsEnabled("WebRTC-DefaultBitrateLimitsKillSwitch")),
+ qp_parsing_allowed_(
+ !field_trial::IsEnabled("WebRTC-QpParsingKillSwitch")),
encoder_queue_(task_queue_factory->CreateTaskQueue(
"EncoderQueue",
TaskQueueFactory::Priority::NORMAL)) {
@@ -1872,6 +1874,18 @@
frame_encode_metadata_writer_.UpdateBitstream(codec_specific_info,
&image_copy);
+ VideoCodecType codec_type = codec_specific_info
+ ? codec_specific_info->codecType
+ : VideoCodecType::kVideoCodecGeneric;
+
+ if (image_copy.qp_ < 0 && qp_parsing_allowed_) {
+ // Parse encoded frame QP if that was not provided by encoder.
+ image_copy.qp_ = qp_parser_
+ .Parse(codec_type, spatial_idx, image_copy.data(),
+ image_copy.size())
+ .value_or(-1);
+ }
+
// Piggyback ALR experiment group id and simulcast id into the content type.
const uint8_t experiment_id =
experiment_groups_[videocontenttypehelpers::IsScreenshare(
@@ -1894,12 +1908,9 @@
// Post a task because |send_codec_| requires |encoder_queue_| lock.
unsigned int image_width = image_copy._encodedWidth;
unsigned int image_height = image_copy._encodedHeight;
- VideoCodecType codec = codec_specific_info
- ? codec_specific_info->codecType
- : VideoCodecType::kVideoCodecGeneric;
- encoder_queue_.PostTask([this, codec, image_width, image_height] {
+ encoder_queue_.PostTask([this, codec_type, image_width, image_height] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- if (codec == VideoCodecType::kVideoCodecVP9 &&
+ if (codec_type == VideoCodecType::kVideoCodecVP9 &&
send_codec_.VP9()->automaticResizeOn) {
unsigned int expected_width = send_codec_.width;
unsigned int expected_height = send_codec_.height;
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index b1c3bd8..e97dc21 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -34,6 +34,7 @@
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_input_state_provider.h"
#include "modules/video_coding/utility/frame_dropper.h"
+#include "modules/video_coding/utility/qp_parser.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/race_checker.h"
@@ -443,6 +444,11 @@
// Default bitrate limits in EncoderInfoSettings allowed.
const bool default_limits_allowed_;
+ // QP parser is used to extract QP value from encoded frame when that is not
+ // provided by encoder.
+ QpParser qp_parser_;
+ const bool qp_parsing_allowed_;
+
// Public methods are proxied to the task queues. The queues must be destroyed
// first to make sure no tasks run that use other members.
rtc::TaskQueue encoder_queue_;
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 72ff984..98e63a7 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -97,6 +97,11 @@
0x05, 0x03, 0xC7, 0xE0, 0x1B,
0x41, 0x10, 0x8D, 0x00};
+const uint8_t kCodedFrameVp8Qp25[] = {
+ 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
+ 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
+ 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
+
class TestBuffer : public webrtc::I420Buffer {
public:
TestBuffer(rtc::Event* event, int width, int height)
@@ -960,9 +965,10 @@
FakeEncoder::Encode(input_image, &frame_type);
}
- void InjectEncodedImage(const EncodedImage& image) {
+ void InjectEncodedImage(const EncodedImage& image,
+ const CodecSpecificInfo* codec_specific_info) {
MutexLock lock(&local_mutex_);
- encoded_image_callback_->OnEncodedImage(image, nullptr);
+ encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
}
void SetEncodedImageData(
@@ -1248,6 +1254,11 @@
return last_capture_time_ms_;
}
+ const EncodedImage& GetLastEncodedImage() {
+ MutexLock lock(&mutex_);
+ return last_encoded_image_;
+ }
+
std::vector<uint8_t> GetLastEncodedImageData() {
MutexLock lock(&mutex_);
return std::move(last_encoded_image_data_);
@@ -1279,6 +1290,7 @@
const CodecSpecificInfo* codec_specific_info) override {
MutexLock lock(&mutex_);
EXPECT_TRUE(expect_frames_);
+ last_encoded_image_ = EncodedImage(encoded_image);
last_encoded_image_data_ = std::vector<uint8_t>(
encoded_image.data(), encoded_image.data() + encoded_image.size());
uint32_t timestamp = encoded_image.Timestamp();
@@ -1337,6 +1349,7 @@
mutable Mutex mutex_;
TestEncoder* test_encoder_;
rtc::Event encoded_frame_event_;
+ EncodedImage last_encoded_image_;
std::vector<uint8_t> last_encoded_image_data_;
uint32_t last_timestamp_ = 0;
int64_t last_capture_time_ms_ = 0;
@@ -7135,14 +7148,12 @@
int64_t timestamp = 1;
EncodedImage image;
- image.SetEncodedData(
- EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
image.capture_time_ms_ = ++timestamp;
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
const int64_t kEncodeFinishDelayMs = 10;
image.timing_.encode_start_ms = timestamp;
image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
- fake_encoder_.InjectEncodedImage(image);
+ fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
// Wait for frame without incrementing clock.
EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
// Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
@@ -7922,4 +7933,63 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
+ // Enable encoder source to force encoder reconfig.
+ encoder_factory_.SetHasInternalSource(true);
+ ResetEncoder("VP8", 1, 1, 1, false);
+
+ // Set QP on encoded frame and pass the frame to encode complete callback.
+ // Since QP is present QP parsing won't be triggered and the original value
+ // should be kept.
+ EncodedImage encoded_image;
+ encoded_image.qp_ = 123;
+ encoded_image.SetEncodedData(EncodedImageBuffer::Create(
+ kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+ CodecSpecificInfo codec_info;
+ codec_info.codecType = kVideoCodecVP8;
+ fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
+ EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
+ EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
+ // Enable encoder source to force encoder reconfig.
+ encoder_factory_.SetHasInternalSource(true);
+ ResetEncoder("VP8", 1, 1, 1, false);
+
+ // Pass an encoded frame without QP to encode complete callback. QP should be
+ // parsed and set.
+ EncodedImage encoded_image;
+ encoded_image.qp_ = -1;
+ encoded_image.SetEncodedData(EncodedImageBuffer::Create(
+ kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+ CodecSpecificInfo codec_info;
+ codec_info.codecType = kVideoCodecVP8;
+ fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
+ EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
+ EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-QpParsingKillSwitch/Enabled/");
+
+ // Enable encoder source to force encoder reconfig.
+ encoder_factory_.SetHasInternalSource(true);
+ ResetEncoder("VP8", 1, 1, 1, false);
+
+ EncodedImage encoded_image;
+ encoded_image.qp_ = -1;
+ encoded_image.SetEncodedData(EncodedImageBuffer::Create(
+ kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+ CodecSpecificInfo codec_info;
+ codec_info.codecType = kVideoCodecVP8;
+ fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
+ EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
+ EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
+ video_stream_encoder_->Stop();
+}
+
} // namespace webrtc
