Add FrameInstrumentationGenerator to VideoStreamEncoder
The FrameInstrumentationGenerator is responsible for generating the
instrumentation data that will be used to detect corruption. The data is
then passed to the encoder in the CodecSpecificInfo.
Bug: webrtc:358039777
Change-Id: I79d0534920b4c7fa001e1138371dfd36c13424fb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/362584
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Fanny Linderborg <linderborg@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43060}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 2f4d833..0eb6d6f 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -425,6 +425,7 @@
"../api/video_codecs:video_codecs_api",
"../call/adaptation:resource_adaptation",
"../common_video",
+ "../common_video:frame_instrumentation_data",
"../media:media_channel",
"../modules:module_api_public",
"../modules/video_coding",
@@ -461,9 +462,11 @@
"adaptation:video_adaptation",
"config:encoder_config",
"config:streams_config",
+ "corruption_detection:frame_instrumentation_generator",
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/cleanup",
"//third_party/abseil-cpp/absl/container:inlined_vector",
+ "//third_party/abseil-cpp/absl/types:variant",
]
}
@@ -873,6 +876,7 @@
"../call/adaptation:resource_adaptation",
"../call/adaptation:resource_adaptation_test_utilities",
"../common_video",
+ "../common_video:frame_instrumentation_data",
"../common_video/test:utilities",
"../media:codec",
"../media:media_constants",
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index da42b2a..4fc7771 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -20,6 +20,7 @@
#include "absl/algorithm/container.h"
#include "absl/cleanup/cleanup.h"
+#include "absl/types/variant.h"
#include "api/field_trials_view.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
@@ -30,13 +31,16 @@
#include "api/video/video_bitrate_allocator_factory.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_layers_allocation.h"
+#include "api/video/video_stream_encoder_settings.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource_adaptation_processor.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_adapter.h"
+#include "common_video/frame_instrumentation_data.h"
#include "media/base/media_channel.h"
#include "modules/video_coding/include/video_codec_initializer.h"
+#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/svc/scalability_mode_util.h"
#include "modules/video_coding/svc/svc_rate_allocator.h"
#include "modules/video_coding/utility/vp8_constants.h"
@@ -54,6 +58,8 @@
#include "video/adaptation/video_stream_encoder_resource_manager.h"
#include "video/alignment_adjuster.h"
#include "video/config/encoder_stream_factory.h"
+#include "video/config/video_encoder_config.h"
+#include "video/corruption_detection/frame_instrumentation_generator.h"
#include "video/frame_cadence_adapter.h"
#include "video/frame_dumping_encoder.h"
@@ -757,6 +763,7 @@
ReleaseEncoder();
encoder_ = nullptr;
frame_cadence_adapter_ = nullptr;
+ frame_instrumentation_generator_ = nullptr;
});
shutdown_event.Wait(rtc::Event::kForever);
}
@@ -934,6 +941,12 @@
max_data_payload_length_ = max_data_payload_length;
pending_encoder_reconfiguration_ = true;
+ if (settings_.enable_frame_instrumentation_generator) {
+ frame_instrumentation_generator_ =
+ std::make_unique<FrameInstrumentationGenerator>(
+ encoder_config_.codec_type);
+ }
+
// Reconfigure the encoder now if the frame resolution is known.
// Otherwise, the reconfiguration is deferred until the next frame to
// minimize the number of reconfigurations. The codec configuration
@@ -1532,6 +1545,9 @@
encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(),
incoming_frame.height());
+ if (frame_instrumentation_generator_) {
+ frame_instrumentation_generator_->OnCapturedFrame(incoming_frame);
+ }
++captured_frame_count_;
bool cwnd_frame_drop =
cwnd_frame_drop_interval_ &&
@@ -2164,6 +2180,22 @@
// running in parallel on different threads.
encoder_stats_observer_->OnSendEncodedImage(image_copy, codec_specific_info);
+ std::unique_ptr<CodecSpecificInfo> codec_specific_info_copy;
+ if (codec_specific_info && frame_instrumentation_generator_) {
+ std::optional<
+ absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+ frame_instrumentation_data =
+ frame_instrumentation_generator_->OnEncodedImage(image_copy);
+ RTC_CHECK(!codec_specific_info->frame_instrumentation_data.has_value())
+ << "CodecSpecificInfo must not have frame_instrumentation_data set.";
+ if (frame_instrumentation_data.has_value()) {
+ codec_specific_info_copy =
+ std::make_unique<CodecSpecificInfo>(*codec_specific_info);
+ codec_specific_info_copy->frame_instrumentation_data =
+ frame_instrumentation_data;
+ codec_specific_info = codec_specific_info_copy.get();
+ }
+ }
EncodedImageCallback::Result result =
sink_->OnEncodedImage(image_copy, codec_specific_info);
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index dd423c3..475a9e6 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -44,6 +44,7 @@
#include "rtc_base/rate_statistics.h"
#include "rtc_base/thread_annotations.h"
#include "video/adaptation/video_stream_encoder_resource_manager.h"
+#include "video/corruption_detection/frame_instrumentation_generator.h"
#include "video/encoder_bitrate_adjuster.h"
#include "video/frame_cadence_adapter.h"
#include "video/frame_encode_metadata_writer.h"
@@ -447,6 +448,10 @@
ScopedTaskSafety task_safety_;
std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue_;
+
+ // Required for automatic corruption detection.
+ std::unique_ptr<FrameInstrumentationGenerator>
+ frame_instrumentation_generator_;
};
} // namespace webrtc
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 9e0626c..5fe7373 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -1,4 +1,3 @@
-
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
@@ -14,10 +13,12 @@
#include <cstdint>
#include <limits>
#include <memory>
+#include <optional>
#include <tuple>
#include <utility>
#include "absl/memory/memory.h"
+#include "absl/types/variant.h"
#include "api/environment/environment.h"
#include "api/environment/environment_factory.h"
#include "api/field_trials_view.h"
@@ -31,6 +32,7 @@
#include "api/units/data_rate.h"
#include "api/units/time_delta.h"
#include "api/video/builtin_video_bitrate_allocator_factory.h"
+#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/nv12_buffer.h"
#include "api/video/resolution.h"
@@ -44,6 +46,7 @@
#include "api/video_codecs/vp8_temporal_layers_factory.h"
#include "call/adaptation/test/fake_adaptation_constraint.h"
#include "call/adaptation/test/fake_resource.h"
+#include "common_video/frame_instrumentation_data.h"
#include "common_video/h264/h264_common.h"
#include "common_video/include/video_frame_buffer.h"
#include "media/base/video_adapter.h"
@@ -54,6 +57,7 @@
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
#include "modules/video_coding/codecs/vp9/svc_config.h"
+#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/quality_scaler.h"
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include "modules/video_coding/utility/vp8_constants.h"
@@ -63,6 +67,7 @@
#include "rtc_base/logging.h"
#include "rtc_base/ref_counted_object.h"
#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
#include "system_wrappers/include/metrics.h"
#include "test/encoder_settings.h"
#include "test/fake_encoder.h"
@@ -1513,6 +1518,13 @@
return number_of_layers_allocations_;
}
+ std::optional<
+ absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+ GetLastFrameInstrumentationData() const {
+ MutexLock lock(&mutex_);
+ return last_frame_instrumentation_data_;
+ }
+
private:
Result OnEncodedImage(
const EncodedImage& encoded_image,
@@ -1538,6 +1550,11 @@
if (num_received_layers_ == num_expected_layers_) {
encoded_frame_event_.Set();
}
+ if (codec_specific_info &&
+ codec_specific_info->frame_instrumentation_data.has_value()) {
+ last_frame_instrumentation_data_ =
+ codec_specific_info->frame_instrumentation_data;
+ }
return Result(Result::OK, last_timestamp_);
}
@@ -1597,6 +1614,9 @@
int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
+ std::optional<
+ absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+ last_frame_instrumentation_data_ RTC_GUARDED_BY(&mutex_);
};
class VideoBitrateAllocatorProxyFactory
@@ -1664,6 +1684,44 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest, PopulatesFrameInstrumentationDataWhenSetTo) {
+ video_send_config_.encoder_settings.enable_frame_instrumentation_generator =
+ true;
+ ConfigureEncoder(video_encoder_config_.Copy());
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // We need a QP for the encoded frame.
+ fake_encoder_.SetEncodedImageData(EncodedImageBuffer::Create(
+ kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, codec_width_, codec_height_));
+ WaitForEncodedFrame(1);
+
+ EXPECT_TRUE(sink_.GetLastFrameInstrumentationData().has_value());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ DoesNotPopulateFrameInstrumentationDataWhenSetNotTo) {
+ video_send_config_.encoder_settings.enable_frame_instrumentation_generator =
+ false;
+ ConfigureEncoder(video_encoder_config_.Copy());
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // We need a QP for the encoded frame.
+ fake_encoder_.SetEncodedImageData(EncodedImageBuffer::Create(
+ kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
+ WaitForEncodedFrame(1);
+
+ EXPECT_FALSE(sink_.GetLastFrameInstrumentationData().has_value());
+ video_stream_encoder_->Stop();
+ EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
+}
+
TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
// Dropped since no target bitrate has been set.
rtc::Event frame_destroyed_event;
