Add optional corruption filter settings to EncodedImage.
This is a prerequisite for enabling implementation-specific filter
settings for automatic corruption detection.
Bug: webrtc:358039777
Change-Id: I363c592aa35164f690dd4ad1204e90afc0277d8b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/368940
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43443}
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index fd6cbd2..57e4d9f 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -142,6 +142,11 @@
deps = [ "../../rtc_base/system:rtc_export" ]
}
+rtc_source_set("corruption_detection_filter_settings") {
+ visibility = [ "*" ]
+ public = [ "corruption_detection_filter_settings.h" ]
+}
+
rtc_library("encoded_image") {
visibility = [ "*" ]
sources = [
@@ -149,6 +154,7 @@
"encoded_image.h",
]
deps = [
+ ":corruption_detection_filter_settings",
":video_codec_constants",
":video_frame",
":video_frame_type",
diff --git a/api/video/corruption_detection_filter_settings.h b/api/video/corruption_detection_filter_settings.h
new file mode 100644
index 0000000..9ae2414
--- /dev/null
+++ b/api/video/corruption_detection_filter_settings.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_
+#define API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_
+
+#include <stdint.h>
+
+// Filter settings for automatic corruption detection. See
+// http://www.webrtc.org/experiments/rtp-hdrext/corruption-detection for more
+// information.
+struct CorruptionDetectionFilterSettings {
+ // Size of the blur kernel used.
+ double std_dev = 0.0;
+ // Allowed error thresholds (maps to `Y err` and `UV err` respectively).
+ int luma_error_threshold = 0;
+ int chroma_error_threshold = 0;
+};
+
+#endif // API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_
diff --git a/api/video/encoded_image.h b/api/video/encoded_image.h
index 53264be..553768e 100644
--- a/api/video/encoded_image.h
+++ b/api/video/encoded_image.h
@@ -23,6 +23,7 @@
#include "api/scoped_refptr.h"
#include "api/units/timestamp.h"
#include "api/video/color_space.h"
+#include "api/video/corruption_detection_filter_settings.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_content_type.h"
#include "api/video/video_frame_type.h"
@@ -228,6 +229,15 @@
VideoContentType contentType() const { return content_type_; }
VideoRotation rotation() const { return rotation_; }
+ std::optional<CorruptionDetectionFilterSettings>
+ corruption_detection_filter_settings() const {
+ return corruption_detection_filter_settings_;
+ }
+ void set_corruption_detection_filter_settings(
+ const CorruptionDetectionFilterSettings& settings) {
+ corruption_detection_filter_settings_ = settings;
+ }
+
uint32_t _encodedWidth = 0;
uint32_t _encodedHeight = 0;
// NTP time of the capture time in local timebase in milliseconds.
@@ -283,6 +293,12 @@
// True if the frame that was encoded is a steady-state refresh frame intended
// to improve the visual quality.
bool is_steady_state_refresh_frame_ = false;
+
+ // Filter settings for corruption detection suggested by the encoder
+ // implementation, if any. Otherwise generic per-codec-type settings will be
+ // used.
+ std::optional<CorruptionDetectionFilterSettings>
+ corruption_detection_filter_settings_;
};
} // namespace webrtc
diff --git a/video/corruption_detection/BUILD.gn b/video/corruption_detection/BUILD.gn
index c7ab70b..9f98783 100644
--- a/video/corruption_detection/BUILD.gn
+++ b/video/corruption_detection/BUILD.gn
@@ -47,6 +47,7 @@
":generic_mapping_functions",
":halton_frame_sampler",
"../../api:scoped_refptr",
+ "../../api/video:corruption_detection_filter_settings",
"../../api/video:encoded_image",
"../../api/video:video_frame",
"../../api/video:video_frame_type",
@@ -84,6 +85,7 @@
"generic_mapping_functions.h",
]
deps = [
+ "../../api/video:corruption_detection_filter_settings",
"../../api/video:video_frame",
"../../api/video_codecs:video_codecs_api",
"../../rtc_base:checks",
diff --git a/video/corruption_detection/frame_instrumentation_generator.cc b/video/corruption_detection/frame_instrumentation_generator.cc
index df6fe86..508afc6 100644
--- a/video/corruption_detection/frame_instrumentation_generator.cc
+++ b/video/corruption_detection/frame_instrumentation_generator.cc
@@ -19,6 +19,7 @@
#include "absl/algorithm/container.h"
#include "absl/types/variant.h"
#include "api/scoped_refptr.h"
+#include "api/video/corruption_detection_filter_settings.h"
#include "api/video/encoded_image.h"
#include "api/video/video_codec_type.h"
#include "api/video/video_frame.h"
@@ -40,36 +41,33 @@
// can lead to frame buffer pools draining.
constexpr size_t kMaxPendingFrames = 3;
-std::optional<FilterSettings> GetCorruptionFilterSettings(
+std::optional<CorruptionDetectionFilterSettings> GetCorruptionFilterSettings(
const EncodedImage& encoded_image,
VideoCodecType video_codec_type,
int layer_id) {
- /* TODO: bugs.webrtc.org/358039777 - Uncomment when parameters are available
- in EncodedImage.
- if (encoded_image.CorruptionDetectionParameters()) {
- return FilterSettings{
- .std_dev = encoded_image.CorruptionDetectionParameters()->std_dev,
- .luma_error_threshold =
- encoded_image.CorruptionDetectionParameters()->luma_error_threshold,
- .chroma_error_threshold = encoded_image.CorruptionDetectionParameters()
- ->chroma_error_threshold};
- }
- */
+ std::optional<CorruptionDetectionFilterSettings> filter_settings =
+ encoded_image.corruption_detection_filter_settings();
- int qp = encoded_image.qp_;
- if (qp == -1) {
- std::optional<uint32_t> parsed_qp = QpParser().Parse(
- video_codec_type, layer_id, encoded_image.data(), encoded_image.size());
- if (!parsed_qp.has_value()) {
- RTC_LOG(LS_VERBOSE) << "Missing QP for "
- << CodecTypeToPayloadString(video_codec_type)
- << " layer " << layer_id << ".";
- return std::nullopt;
+ if (!filter_settings.has_value()) {
+ // No implementation specific filter settings available, using a generic
+ // QP-based settings instead.
+ int qp = encoded_image.qp_;
+ if (qp == -1) {
+ std::optional<uint32_t> parsed_qp =
+ QpParser().Parse(video_codec_type, layer_id, encoded_image.data(),
+ encoded_image.size());
+ if (!parsed_qp.has_value()) {
+ RTC_LOG(LS_VERBOSE)
+ << "Missing QP for " << CodecTypeToPayloadString(video_codec_type)
+ << " layer " << layer_id << ".";
+ return std::nullopt;
+ }
+ qp = *parsed_qp;
}
- qp = *parsed_qp;
- }
- return GetCorruptionFilterSettings(qp, video_codec_type);
+ filter_settings = GetCorruptionFilterSettings(qp, video_codec_type);
+ }
+ return filter_settings;
}
} // namespace
@@ -158,7 +156,7 @@
.communicate_upper_bits = true};
}
- std::optional<FilterSettings> filter_settings =
+ std::optional<CorruptionDetectionFilterSettings> filter_settings =
GetCorruptionFilterSettings(encoded_image, video_codec_type_, layer_id);
if (!filter_settings.has_value()) {
return std::nullopt;
diff --git a/video/corruption_detection/frame_instrumentation_generator_unittest.cc b/video/corruption_detection/frame_instrumentation_generator_unittest.cc
index 7f8992b..5f3b011 100644
--- a/video/corruption_detection/frame_instrumentation_generator_unittest.cc
+++ b/video/corruption_detection/frame_instrumentation_generator_unittest.cc
@@ -722,5 +722,37 @@
EXPECT_THAT(frames_destroyed, ElementsAre(true, true, true, true));
}
+TEST(FrameInstrumentationGeneratorTest,
+ UsesFilterSettingsFromFrameWhenAvailable) {
+ FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8);
+ VideoFrame frame = VideoFrame::Builder()
+ .set_video_frame_buffer(MakeDefaultI420FrameBuffer())
+ .set_rtp_timestamp(1)
+ .build();
+ // No QP needed when frame provides filter settings.
+ EncodedImage encoded_image;
+ encoded_image.SetRtpTimestamp(1);
+ encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey);
+ encoded_image._encodedWidth = kDefaultScaledWidth;
+ encoded_image._encodedHeight = kDefaultScaledHeight;
+ encoded_image.set_corruption_detection_filter_settings(
+ CorruptionDetectionFilterSettings{.std_dev = 1.0,
+ .luma_error_threshold = 2,
+ .chroma_error_threshold = 3});
+
+ generator.OnCapturedFrame(frame);
+ std::optional<
+ absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+ data = generator.OnEncodedImage(encoded_image);
+
+ ASSERT_TRUE(data.has_value());
+ ASSERT_TRUE(absl::holds_alternative<FrameInstrumentationData>(*data));
+ FrameInstrumentationData frame_instrumentation_data =
+ absl::get<FrameInstrumentationData>(*data);
+ EXPECT_EQ(frame_instrumentation_data.std_dev, 1.0);
+ EXPECT_EQ(frame_instrumentation_data.luma_error_threshold, 2);
+ EXPECT_EQ(frame_instrumentation_data.chroma_error_threshold, 3);
+}
+
} // namespace
} // namespace webrtc
diff --git a/video/corruption_detection/frame_pair_corruption_score.cc b/video/corruption_detection/frame_pair_corruption_score.cc
index 084005a..91b789c 100644
--- a/video/corruption_detection/frame_pair_corruption_score.cc
+++ b/video/corruption_detection/frame_pair_corruption_score.cc
@@ -77,7 +77,8 @@
scoped_refptr<I420Buffer> test_i420_buffer =
GetAsI420Buffer(test_buffer.ToI420());
- FilterSettings filter_settings = GetCorruptionFilterSettings(qp, codec_type_);
+ CorruptionDetectionFilterSettings filter_settings =
+ GetCorruptionFilterSettings(qp, codec_type_);
const std::vector<FilteredSample> filtered_reference_sample_values =
GetSampleValuesForFrame(
diff --git a/video/corruption_detection/generic_mapping_functions.cc b/video/corruption_detection/generic_mapping_functions.cc
index ff37734..cb489b8 100644
--- a/video/corruption_detection/generic_mapping_functions.cc
+++ b/video/corruption_detection/generic_mapping_functions.cc
@@ -73,10 +73,13 @@
} // namespace
-FilterSettings GetCorruptionFilterSettings(int qp, VideoCodecType codec_type) {
- return FilterSettings{.std_dev = MapQpToOptimalStdDev(qp, codec_type),
- .luma_error_threshold = LumaThreshold(codec_type),
- .chroma_error_threshold = ChromaThreshold(codec_type)};
+CorruptionDetectionFilterSettings GetCorruptionFilterSettings(
+ int qp,
+ VideoCodecType codec_type) {
+ return CorruptionDetectionFilterSettings{
+ .std_dev = MapQpToOptimalStdDev(qp, codec_type),
+ .luma_error_threshold = LumaThreshold(codec_type),
+ .chroma_error_threshold = ChromaThreshold(codec_type)};
}
} // namespace webrtc
diff --git a/video/corruption_detection/generic_mapping_functions.h b/video/corruption_detection/generic_mapping_functions.h
index 81100df..191c229 100644
--- a/video/corruption_detection/generic_mapping_functions.h
+++ b/video/corruption_detection/generic_mapping_functions.h
@@ -11,17 +11,17 @@
#ifndef VIDEO_CORRUPTION_DETECTION_GENERIC_MAPPING_FUNCTIONS_H_
#define VIDEO_CORRUPTION_DETECTION_GENERIC_MAPPING_FUNCTIONS_H_
+#include "api/video/corruption_detection_filter_settings.h"
#include "api/video/video_codec_type.h"
namespace webrtc {
-struct FilterSettings {
- double std_dev = 0.0;
- int luma_error_threshold = 0;
- int chroma_error_threshold = 0;
-};
+// TODO: bugs.webrtc.org/358039777 - Remove when downstream usage is gone.
+using FilterSettings = CorruptionDetectionFilterSettings;
-FilterSettings GetCorruptionFilterSettings(int qp, VideoCodecType codec_type);
+CorruptionDetectionFilterSettings GetCorruptionFilterSettings(
+ int qp,
+ VideoCodecType codec_type);
} // namespace webrtc
