Add helper for comparing FrameInstrumentationData with a VideoFrame
Bug: webrtc:358039777
Change-Id: Ibe597160658dbc66aba427f4e30dade4d6fe56e2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/363701
Auto-Submit: Fanny Linderborg <linderborg@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43134}
diff --git a/video/corruption_detection/BUILD.gn b/video/corruption_detection/BUILD.gn
index d37e359..55ab651 100644
--- a/video/corruption_detection/BUILD.gn
+++ b/video/corruption_detection/BUILD.gn
@@ -21,6 +21,23 @@
]
}
+rtc_library("frame_instrumentation_evaluation") {
+ sources = [
+ "frame_instrumentation_evaluation.cc",
+ "frame_instrumentation_evaluation.h",
+ ]
+ deps = [
+ ":corruption_classifier",
+ ":halton_frame_sampler",
+ "../../api:array_view",
+ "../../api:scoped_refptr",
+ "../../api/video:video_frame",
+ "../../common_video:frame_instrumentation_data",
+ "../../rtc_base:checks",
+ "../../rtc_base:logging",
+ ]
+}
+
rtc_library("frame_instrumentation_generator") {
sources = [
"frame_instrumentation_generator.cc",
@@ -89,6 +106,18 @@
]
}
+ rtc_library("frame_instrumentation_evaluation_unittest") {
+ testonly = true
+ sources = [ "frame_instrumentation_evaluation_unittest.cc" ]
+ deps = [
+ ":frame_instrumentation_evaluation",
+ "../../api:scoped_refptr",
+ "../../api/video:video_frame",
+ "../../common_video:frame_instrumentation_data",
+ "../../test:test_support",
+ ]
+ }
+
rtc_library("frame_instrumentation_generator_unittest") {
testonly = true
sources = [ "frame_instrumentation_generator_unittest.cc" ]
@@ -139,6 +168,7 @@
sources = []
deps = [
":corruption_classifier_unittest",
+ ":frame_instrumentation_evaluation_unittest",
":frame_instrumentation_generator_unittest",
":generic_mapping_functions_unittest",
":halton_frame_sampler_unittest",
diff --git a/video/corruption_detection/frame_instrumentation_evaluation.cc b/video/corruption_detection/frame_instrumentation_evaluation.cc
new file mode 100644
index 0000000..f3367ad
--- /dev/null
+++ b/video/corruption_detection/frame_instrumentation_evaluation.cc
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2024 The WebRTC project authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/corruption_detection/frame_instrumentation_evaluation.h"
+
+#include <cstddef>
+#include <optional>
+#include <vector>
+
+#include "api/array_view.h"
+#include "api/scoped_refptr.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_buffer.h"
+#include "common_video/frame_instrumentation_data.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "video/corruption_detection/corruption_classifier.h"
+#include "video/corruption_detection/halton_frame_sampler.h"
+
+namespace webrtc {
+
+namespace {
+
+std::vector<FilteredSample> ConvertSampleValuesToFilteredSamples(
+ rtc::ArrayView<const double> values,
+ rtc::ArrayView<const FilteredSample> samples) {
+ RTC_CHECK_EQ(values.size(), samples.size())
+ << "values and samples must have the same size";
+ std::vector<FilteredSample> filtered_samples;
+ filtered_samples.reserve(values.size());
+ for (size_t i = 0; i < values.size(); ++i) {
+ filtered_samples.push_back({.value = values[i], .plane = samples[i].plane});
+ }
+ return filtered_samples;
+}
+
+} // namespace
+
+std::optional<double> GetCorruptionScore(const FrameInstrumentationData& data,
+ const VideoFrame& frame) {
+ if (data.sample_values.empty()) {
+ RTC_LOG(LS_WARNING)
+ << "Samples are needed to calculate a corruption score.";
+ return std::nullopt;
+ }
+
+ scoped_refptr<I420BufferInterface> frame_buffer_as_i420 =
+ frame.video_frame_buffer()->ToI420();
+ if (!frame_buffer_as_i420) {
+ RTC_LOG(LS_ERROR) << "Failed to convert "
+ << VideoFrameBufferTypeToString(
+ frame.video_frame_buffer()->type())
+ << " image to I420";
+ return std::nullopt;
+ }
+
+ HaltonFrameSampler frame_sampler;
+ frame_sampler.SetCurrentIndex(data.sequence_index);
+ std::vector<HaltonFrameSampler::Coordinates> sample_coordinates =
+ frame_sampler.GetSampleCoordinatesForFrame(data.sample_values.size());
+ if (sample_coordinates.empty()) {
+ RTC_LOG(LS_ERROR) << "Failed to get sample coordinates for frame.";
+ return std::nullopt;
+ }
+
+ std::vector<FilteredSample> samples =
+ GetSampleValuesForFrame(frame_buffer_as_i420, sample_coordinates,
+ frame.width(), frame.height(), data.std_dev);
+ if (samples.empty()) {
+ RTC_LOG(LS_ERROR) << "Failed to get sample values for frame";
+ return std::nullopt;
+ }
+
+ std::vector<FilteredSample> data_samples =
+ ConvertSampleValuesToFilteredSamples(data.sample_values, samples);
+ if (data_samples.empty()) {
+ RTC_LOG(LS_ERROR) << "Failed to convert sample values to filtered samples";
+ return std::nullopt;
+ }
+
+ // TODO: bugs.webrtc.org/358039777 - Update before rollout. Which variant of
+ // classifier should we use? What input parameters should it have?
+ CorruptionClassifier classifier(2.5);
+
+ return classifier.CalculateCorruptionProbablility(
+ data_samples, samples, data.luma_error_threshold,
+ data.chroma_error_threshold);
+}
+
+} // namespace webrtc
diff --git a/video/corruption_detection/frame_instrumentation_evaluation.h b/video/corruption_detection/frame_instrumentation_evaluation.h
new file mode 100644
index 0000000..8bd3e1c
--- /dev/null
+++ b/video/corruption_detection/frame_instrumentation_evaluation.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2024 The WebRTC project authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_
+#define VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_
+
+#include <optional>
+
+#include "api/video/video_frame.h"
+#include "common_video/frame_instrumentation_data.h"
+
+namespace webrtc {
+
+std::optional<double> GetCorruptionScore(const FrameInstrumentationData& data,
+ const VideoFrame& frame);
+
+} // namespace webrtc
+
+#endif // VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_
diff --git a/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc b/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc
new file mode 100644
index 0000000..cfc17e4
--- /dev/null
+++ b/video/corruption_detection/frame_instrumentation_evaluation_unittest.cc
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2024 The WebRTC project authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/corruption_detection/frame_instrumentation_evaluation.h"
+
+#include <cstdint>
+#include <optional>
+#include <vector>
+
+#include "api/scoped_refptr.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "common_video/frame_instrumentation_data.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+scoped_refptr<I420Buffer> MakeI420FrameBufferWithDifferentPixelValues() {
+ // Create an I420 frame of size 4x4.
+ const int kDefaultLumaWidth = 4;
+ const int kDefaultLumaHeight = 4;
+ const int kDefaultChromaWidth = 2;
+ std::vector<uint8_t> kDefaultYContent = {1, 2, 3, 4, 5, 6, 7, 8,
+ 9, 10, 11, 12, 13, 14, 15, 16};
+ std::vector<uint8_t> kDefaultUContent = {17, 18, 19, 20};
+ std::vector<uint8_t> kDefaultVContent = {21, 22, 23, 24};
+
+ return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight,
+ kDefaultYContent.data(), kDefaultLumaWidth,
+ kDefaultUContent.data(), kDefaultChromaWidth,
+ kDefaultVContent.data(), kDefaultChromaWidth);
+}
+
+TEST(FrameInstrumentationEvaluationTest,
+ HaveNoCorruptionScoreWhenNoSampleValuesAreProvided) {
+ FrameInstrumentationData data = {.sequence_index = 0,
+ .communicate_upper_bits = false,
+ .std_dev = 0.0,
+ .luma_error_threshold = 0,
+ .chroma_error_threshold = 0,
+ .sample_values = {}};
+ VideoFrame frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+ .build();
+
+ std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+ EXPECT_FALSE(corruption_score.has_value());
+}
+
+TEST(FrameInstrumentationEvaluationTest,
+ HaveACorruptionScoreWhenSampleValuesAreProvided) {
+ FrameInstrumentationData data = {
+ .sequence_index = 0,
+ .communicate_upper_bits = false,
+ .std_dev = 0.0,
+ .luma_error_threshold = 0,
+ .chroma_error_threshold = 0,
+ .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}};
+ VideoFrame frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+ .build();
+
+ std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+ ASSERT_TRUE(corruption_score.has_value());
+ EXPECT_DOUBLE_EQ(*corruption_score, 1.0);
+}
+
+TEST(FrameInstrumentationEvaluationTest,
+ ApplyThresholdsWhenNonNegativeThresholdsAreProvided) {
+ FrameInstrumentationData data = {
+ .sequence_index = 0,
+ .communicate_upper_bits = false,
+ .std_dev = 0.0,
+ .luma_error_threshold = 8,
+ .chroma_error_threshold = 8,
+ .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}};
+ VideoFrame frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+ .build();
+
+ std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+ ASSERT_TRUE(corruption_score.has_value());
+ EXPECT_DOUBLE_EQ(*corruption_score, 0.55);
+}
+
+TEST(FrameInstrumentationEvaluationTest,
+ ApplyStdDevWhenNonNegativeStdDevIsProvided) {
+ FrameInstrumentationData data = {
+ .sequence_index = 0,
+ .communicate_upper_bits = false,
+ .std_dev = 0.6,
+ .luma_error_threshold = 8,
+ .chroma_error_threshold = 8,
+ .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}};
+
+ std::vector<double> sample_values = {12, 12, 12, 12, 12, 12, 12, 12};
+ VideoFrame frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+ .build();
+
+ std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+ ASSERT_TRUE(corruption_score.has_value());
+ EXPECT_DOUBLE_EQ(*corruption_score, 0.3302493109581533);
+}
+
+TEST(FrameInstrumentationEvaluationTest, ApplySequenceIndexWhenProvided) {
+ FrameInstrumentationData data = {
+ .sequence_index = 1,
+ .communicate_upper_bits = false,
+ .std_dev = 0.6,
+ .luma_error_threshold = 8,
+ .chroma_error_threshold = 8,
+ .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}};
+
+ std::vector<double> sample_values = {12, 12, 12, 12, 12, 12, 12, 12};
+ VideoFrame frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+ .build();
+
+ std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+ ASSERT_TRUE(corruption_score.has_value());
+ EXPECT_DOUBLE_EQ(*corruption_score, 0.12983429453668965);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/video/corruption_detection/frame_instrumentation_generator.cc b/video/corruption_detection/frame_instrumentation_generator.cc
index b4f0377..cc2b112 100644
--- a/video/corruption_detection/frame_instrumentation_generator.cc
+++ b/video/corruption_detection/frame_instrumentation_generator.cc
@@ -142,7 +142,7 @@
contexts_[layer_id]
.frame_sampler.GetSampleCoordinatesForFrameIfFrameShouldBeSampled(
is_key_frame, captured_frame.rtp_timestamp(),
- /*sample_size=*/13);
+ /*num_samples=*/13);
if (sample_coordinates.empty()) {
if (!is_key_frame) {
return std::nullopt;
