Add video codec tester. This tester is an improved version of VideoProcessor and VideoCodecTestFixture and will eventually replace them. The tester provides better separation between codecs and testing logic. Its knowledge about codecs is limited to frame encode/decode calls and frame ready callbacks. Instantiation and configuration of codecs are the test responsibilities. Other differences: - Run encoding and decoding in separate threads - Run quality analysis in a separate thread - Reference frame buffering is moved into video source (which re-read frames from the file). - Make it possible to run decode-only tests This CL is MVP implementation: it adds only 1 test (video_codec_test.cc, ConstantRate/EncodeDecodeTest) and the test is disabled for now. Bug: b/261160916 Change-Id: Ida24a2fca1b1496237fa695c812084877c76379f Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/283525 Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org> Cr-Commit-Position: refs/heads/main@{#38901}
diff --git a/api/BUILD.gn b/api/BUILD.gn index 7d76433..38ba78f 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn
@@ -985,22 +985,50 @@ ] } - rtc_library("videocodec_test_fixture_api") { + rtc_library("videocodec_test_stats_api") { visibility = [ "*" ] testonly = true sources = [ - "test/videocodec_test_fixture.h", "test/videocodec_test_stats.cc", "test/videocodec_test_stats.h", ] deps = [ - "../modules/video_coding:video_codec_interface", + "../api/units:data_rate", + "../api/units:frequency", "../rtc_base:stringutils", "video:video_frame_type", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("videocodec_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ "test/videocodec_test_fixture.h" ] + deps = [ + ":videocodec_test_stats_api", + "../modules/video_coding:video_codec_interface", "video_codecs:video_codecs_api", ] } + rtc_library("video_codec_tester_api") { + visibility = [ "*" ] + testonly = true + sources = [ "test/video_codec_tester.h" ] + deps = [ + ":videocodec_test_stats_api", + "../modules/video_coding/svc:scalability_mode_util", + "video:encoded_image", + "video:resolution", + "video:video_frame", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + rtc_library("create_videocodec_test_fixture_api") { visibility = [ "*" ] testonly = true @@ -1016,6 +1044,19 @@ ] } + rtc_library("create_video_codec_tester_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/create_video_codec_tester.cc", + "test/create_video_codec_tester.h", + ] + deps = [ + ":video_codec_tester_api", + "../modules/video_coding:videocodec_test_impl", + ] + } + rtc_source_set("mock_audio_mixer") { visibility = [ "*" ] testonly = true
diff --git a/api/test/create_video_codec_tester.cc b/api/test/create_video_codec_tester.cc new file mode 100644 index 0000000..a1efefd --- /dev/null +++ b/api/test/create_video_codec_tester.cc
@@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_video_codec_tester.h" + +#include <memory> +#include <utility> + +#include "api/test/video_codec_tester.h" +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +namespace webrtc { +namespace test { + +std::unique_ptr<VideoCodecTester> CreateVideoCodecTester() { + return std::make_unique<VideoCodecTesterImpl>(); +} + +} // namespace test +} // namespace webrtc
diff --git a/api/test/create_video_codec_tester.h b/api/test/create_video_codec_tester.h new file mode 100644 index 0000000..c68864c --- /dev/null +++ b/api/test/create_video_codec_tester.h
@@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ +#define API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ + +#include <memory> + +#include "api/test/video_codec_tester.h" + +namespace webrtc { +namespace test { + +std::unique_ptr<VideoCodecTester> CreateVideoCodecTester(); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_VIDEO_CODEC_TESTER_H_
diff --git a/api/test/video_codec_tester.h b/api/test/video_codec_tester.h new file mode 100644 index 0000000..0eaaa1b --- /dev/null +++ b/api/test/video_codec_tester.h
@@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_CODEC_TESTER_H_ +#define API_TEST_VIDEO_CODEC_TESTER_H_ + +#include <memory> + +#include "absl/functional/any_invocable.h" +#include "api/test/videocodec_test_stats.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video/video_frame.h" + +namespace webrtc { +namespace test { + +// Interface for a video codec tester. The interface provides minimalistic set +// of data structures that enables implementation of decode-only, encode-only +// and encode-decode tests. +class VideoCodecTester { + public: + // Pacing settings for codec input. + struct PacingSettings { + enum PacingMode { + // Pacing is not used. Frames are sent to codec back-to-back. + kNoPacing, + // Pace with the rate equal to the target video frame rate. Pacing time is + // derived from RTP timestamp. + kRealTime, + // Pace with the explicitly provided rate. + kConstantRate, + }; + PacingMode mode = PacingMode::kNoPacing; + // Pacing rate for `kConstantRate` mode. + Frequency constant_rate = Frequency::Zero(); + }; + + struct DecoderSettings { + PacingSettings pacing; + }; + + struct EncoderSettings { + PacingSettings pacing; + }; + + virtual ~VideoCodecTester() = default; + + // Interface for a raw video frames source. + class RawVideoSource { + public: + virtual ~RawVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional<VideoFrame> PullFrame() = 0; + + // Returns early pulled frame with RTP timestamp equal to `timestamp_rtp`. + virtual VideoFrame GetFrame(uint32_t timestamp_rtp, + Resolution resolution) = 0; + }; + + // Interface for a coded video frames source. + class CodedVideoSource { + public: + virtual ~CodedVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional<EncodedImage> PullFrame() = 0; + }; + + // Interface for a video encoder. + class Encoder { + public: + using EncodeCallback = + absl::AnyInvocable<void(const EncodedImage& encoded_frame)>; + + virtual ~Encoder() = default; + + virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0; + }; + + // Interface for a video decoder. + class Decoder { + public: + using DecodeCallback = + absl::AnyInvocable<void(const VideoFrame& decoded_frame)>; + + virtual ~Decoder() = default; + + virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0; + }; + + // Pulls coded video frames from `video_source` and passes them to `decoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunDecodeTest( + std::unique_ptr<CodedVideoSource> video_source, + std::unique_ptr<Decoder> decoder, + const DecoderSettings& decoder_settings) = 0; + + // Pulls raw video frames from `video_source` and passes them to `encoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunEncodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + const EncoderSettings& encoder_settings) = 0; + + // Pulls raw video frames from `video_source`, passes them to `encoder` and + // then passes encoded frames to `decoder`. Returns `VideoCodecTestStats` + // object that contains collected per-frame metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunEncodeDecodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + std::unique_ptr<Decoder> decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_CODEC_TESTER_H_
diff --git a/api/test/videocodec_test_stats.h b/api/test/videocodec_test_stats.h index a05985a..12c6063 100644 --- a/api/test/videocodec_test_stats.h +++ b/api/test/videocodec_test_stats.h
@@ -18,6 +18,9 @@ #include <string> #include <vector> +#include "absl/types/optional.h" +#include "api/units/data_rate.h" +#include "api/units/frequency.h" #include "api/video/video_frame_type.h" namespace webrtc { @@ -135,11 +138,16 @@ virtual ~VideoCodecTestStats() = default; - virtual std::vector<FrameStatistics> GetFrameStatistics() = 0; + virtual std::vector<FrameStatistics> GetFrameStatistics() const = 0; virtual std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic( size_t first_frame_num, size_t last_frame_num) = 0; + + virtual VideoStatistics CalcVideoStatistic(size_t first_frame, + size_t last_frame, + DataRate target_bitrate, + Frequency target_framerate) = 0; }; } // namespace test
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 2686047..b097daa 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn
@@ -877,6 +877,8 @@ rtc_library("video_codecs_test_framework") { testonly = true sources = [ + "codecs/test/video_codec_analyzer.cc", + "codecs/test/video_codec_analyzer.h", "codecs/test/video_codec_unittest.cc", "codecs/test/video_codec_unittest.h", "codecs/test/videoprocessor.cc", @@ -895,14 +897,17 @@ "../../api:frame_generator_api", "../../api:scoped_refptr", "../../api:sequence_checker", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/task_queue", + "../../api/task_queue:default_task_queue_factory", "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_image", "../../api/video:resolution", "../../api/video:video_bitrate_allocation", "../../api/video:video_bitrate_allocator", "../../api/video:video_bitrate_allocator_factory", + "../../api/video:video_codec_constants", "../../api/video:video_frame", "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", @@ -911,6 +916,7 @@ "../../rtc_base:checks", "../../rtc_base:macromagic", "../../rtc_base:rtc_event", + "../../rtc_base:task_queue_for_test", "../../rtc_base:timeutils", "../../rtc_base/synchronization:mutex", "../../rtc_base/system:no_unique_address", @@ -959,6 +965,8 @@ rtc_library("videocodec_test_impl") { testonly = true sources = [ + "codecs/test/video_codec_tester_impl.cc", + "codecs/test/video_codec_tester_impl.h", "codecs/test/videocodec_test_fixture_impl.cc", "codecs/test/videocodec_test_fixture_impl.h", ] @@ -970,12 +978,20 @@ ":videocodec_test_stats_impl", ":webrtc_vp9_helpers", "../../api:array_view", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", + "../../api/task_queue:default_task_queue_factory", + "../../api/task_queue:task_queue", "../../api/test/metrics:global_metrics_logger_and_exporter", "../../api/test/metrics:metric", "../../api/test/video:function_video_factory", "../../api/transport:field_trial_based_config", + "../../api/units:frequency", + "../../api/units:time_delta", + "../../api/units:timestamp", + "../../api/video:encoded_image", "../../api/video:video_bitrate_allocation", + "../../api/video:video_frame", "../../api/video_codecs:video_codecs_api", "../../api/video_codecs:video_decoder_factory_template", "../../api/video_codecs:video_decoder_factory_template_dav1d_adapter", @@ -994,6 +1010,7 @@ "../../rtc_base:checks", "../../rtc_base:logging", "../../rtc_base:rtc_base_tests_utils", + "../../rtc_base:rtc_event", "../../rtc_base:stringutils", "../../rtc_base:task_queue_for_test", "../../rtc_base:timeutils", @@ -1018,7 +1035,7 @@ "codecs/test/videocodec_test_stats_impl.h", ] deps = [ - "../../api:videocodec_test_fixture_api", + "../../api:videocodec_test_stats_api", "../../api/numerics", "../../rtc_base:checks", "../../rtc_base:rtc_numerics", @@ -1035,6 +1052,7 @@ sources = [ "codecs/h264/test/h264_impl_unittest.cc", "codecs/multiplex/test/multiplex_adapter_unittest.cc", + "codecs/test/video_codec_test.cc", "codecs/test/video_encoder_decoder_instantiation_tests.cc", "codecs/test/videocodec_test_av1.cc", "codecs/test/videocodec_test_libvpx.cc", @@ -1063,18 +1081,27 @@ ":webrtc_vp9", ":webrtc_vp9_helpers", "../../api:create_frame_generator", + "../../api:create_video_codec_tester_api", "../../api:create_videocodec_test_fixture_api", "../../api:frame_generator_api", "../../api:mock_video_codec_factory", "../../api:mock_video_decoder", "../../api:mock_video_encoder", "../../api:scoped_refptr", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", + "../../api:videocodec_test_stats_api", "../../api/test/video:function_video_factory", + "../../api/units:data_rate", + "../../api/units:frequency", "../../api/video:encoded_image", + "../../api/video:resolution", "../../api/video:video_frame", "../../api/video:video_rtp_headers", + "../../api/video_codecs:builtin_video_decoder_factory", + "../../api/video_codecs:builtin_video_encoder_factory", "../../api/video_codecs:rtc_software_fallback_wrappers", + "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../common_video", "../../common_video/test:utilities", @@ -1090,11 +1117,14 @@ "../../test:fileutils", "../../test:test_support", "../../test:video_test_common", + "../../test:video_test_support", "../rtp_rtcp:rtp_rtcp_format", "codecs/av1:dav1d_decoder", + "svc:scalability_mode_util", "//third_party/libyuv", ] absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/types:optional", ] @@ -1130,6 +1160,8 @@ sources = [ "chain_diff_calculator_unittest.cc", + "codecs/test/video_codec_analyzer_unittest.cc", + "codecs/test/video_codec_tester_impl_unittest.cc", "codecs/test/videocodec_test_fixture_config_unittest.cc", "codecs/test/videocodec_test_stats_impl_unittest.cc", "codecs/test/videoprocessor_unittest.cc", @@ -1213,9 +1245,11 @@ "../../api:rtp_packet_info", "../../api:scoped_refptr", "../../api:simulcast_test_fixture_api", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/task_queue", "../../api/task_queue:default_task_queue_factory", + "../../api/task_queue/test:mock_task_queue_base", "../../api/test/video:function_video_factory", "../../api/units:data_size", "../../api/units:frequency", @@ -1223,6 +1257,7 @@ "../../api/units:timestamp", "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_frame", + "../../api/video:encoded_image", "../../api/video:render_resolution", "../../api/video:video_adaptation", "../../api/video:video_bitrate_allocation", @@ -1239,6 +1274,7 @@ "../../media:rtc_media_base", "../../rtc_base", "../../rtc_base:checks", + "../../rtc_base:gunit_helpers", "../../rtc_base:histogram_percentile_counter", "../../rtc_base:platform_thread", "../../rtc_base:random", @@ -1265,6 +1301,7 @@ "../../test:video_test_common", "../../test:video_test_support", "../../test/time_controller:time_controller", + "../../third_party/libyuv:libyuv", "../rtp_rtcp:rtp_rtcp_format", "../rtp_rtcp:rtp_video_header", "codecs/av1:video_coding_codecs_av1_tests",
diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.cc b/modules/video_coding/codecs/test/video_codec_analyzer.cc new file mode 100644 index 0000000..50af417 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer.cc
@@ -0,0 +1,186 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" + +#include <memory> + +#include "api/task_queue/default_task_queue_factory.h" +#include "api/test/video_codec_tester.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_constants.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/time_utils.h" +#include "third_party/libyuv/include/libyuv/compare.h" + +namespace webrtc { +namespace test { + +namespace { + +struct Psnr { + double y; + double u; + double v; + double yuv; +}; + +Psnr CalcPsnr(const I420BufferInterface& ref_buffer, + const I420BufferInterface& dec_buffer) { + RTC_CHECK_EQ(ref_buffer.width(), dec_buffer.width()); + RTC_CHECK_EQ(ref_buffer.height(), dec_buffer.height()); + + uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), + ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); + + uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), + ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), + ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + int num_y_samples = dec_buffer.width() * dec_buffer.height(); + Psnr psnr; + psnr.y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); + psnr.u = libyuv::SumSquareErrorToPsnr(sse_u, num_y_samples / 4); + psnr.v = libyuv::SumSquareErrorToPsnr(sse_v, num_y_samples / 4); + psnr.yuv = libyuv::SumSquareErrorToPsnr(sse_y + sse_u + sse_v, + num_y_samples + num_y_samples / 2); + return psnr; +} + +} // namespace + +VideoCodecAnalyzer::VideoCodecAnalyzer( + rtc::TaskQueue& task_queue, + ReferenceVideoSource* reference_video_source) + : task_queue_(task_queue), reference_video_source_(reference_video_source) { + sequence_checker_.Detach(); +} + +void VideoCodecAnalyzer::StartEncode(const VideoFrame& input_frame) { + int64_t encode_started_ns = rtc::TimeNanos(); + task_queue_.PostTask( + [this, timestamp_rtp = input_frame.timestamp(), encode_started_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, /*spatial_idx=*/0); + fs->encode_start_ns = encode_started_ns; + }); +} + +void VideoCodecAnalyzer::FinishEncode(const EncodedImage& frame) { + int64_t encode_finished_ns = rtc::TimeNanos(); + + task_queue_.PostTask([this, timestamp_rtp = frame.Timestamp(), + spatial_idx = frame.SpatialIndex().value_or(0), + temporal_idx = frame.TemporalIndex().value_or(0), + frame_type = frame._frameType, qp = frame.qp_, + frame_size_bytes = frame.size(), encode_finished_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, spatial_idx); + VideoCodecTestStats::FrameStatistics* fs_base = + stats_.GetOrAddFrame(timestamp_rtp, 0); + + fs->encode_start_ns = fs_base->encode_start_ns; + fs->spatial_idx = spatial_idx; + fs->temporal_idx = temporal_idx; + fs->frame_type = frame_type; + fs->qp = qp; + + fs->encode_time_us = (encode_finished_ns - fs->encode_start_ns) / + rtc::kNumNanosecsPerMicrosec; + fs->length_bytes = frame_size_bytes; + + fs->encoding_successful = true; + }); +} + +void VideoCodecAnalyzer::StartDecode(const EncodedImage& frame) { + int64_t decode_start_ns = rtc::TimeNanos(); + task_queue_.PostTask([this, timestamp_rtp = frame.Timestamp(), + spatial_idx = frame.SpatialIndex().value_or(0), + frame_size_bytes = frame.size(), decode_start_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, spatial_idx); + if (fs->length_bytes == 0) { + // In encode-decode test the frame size is set in EncodeFinished. In + // decode-only test set it here. + fs->length_bytes = frame_size_bytes; + } + fs->decode_start_ns = decode_start_ns; + }); +} + +void VideoCodecAnalyzer::FinishDecode(const VideoFrame& frame, + int spatial_idx) { + int64_t decode_finished_ns = rtc::TimeNanos(); + task_queue_.PostTask([this, timestamp_rtp = frame.timestamp(), spatial_idx, + width = frame.width(), height = frame.height(), + decode_finished_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + fs->decode_time_us = (decode_finished_ns - fs->decode_start_ns) / + rtc::kNumNanosecsPerMicrosec; + fs->decoded_width = width; + fs->decoded_height = height; + fs->decoding_successful = true; + }); + + if (reference_video_source_ != nullptr) { + // Copy hardware-backed frame into main memory to release output buffers + // which number may be limited in hardware decoders. + rtc::scoped_refptr<I420BufferInterface> decoded_buffer = + frame.video_frame_buffer()->ToI420(); + + task_queue_.PostTask([this, decoded_buffer, + timestamp_rtp = frame.timestamp(), spatial_idx]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoFrame ref_frame = reference_video_source_->GetFrame( + timestamp_rtp, {.width = decoded_buffer->width(), + .height = decoded_buffer->height()}); + rtc::scoped_refptr<I420BufferInterface> ref_buffer = + ref_frame.video_frame_buffer()->ToI420(); + + Psnr psnr = CalcPsnr(*decoded_buffer, *ref_buffer); + VideoCodecTestStats::FrameStatistics* fs = + this->stats_.GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + fs->psnr_y = static_cast<float>(psnr.y); + fs->psnr_u = static_cast<float>(psnr.u); + fs->psnr_v = static_cast<float>(psnr.v); + fs->psnr = static_cast<float>(psnr.yuv); + + fs->quality_analysis_successful = true; + }); + } +} + +std::unique_ptr<VideoCodecTestStats> VideoCodecAnalyzer::GetStats() { + std::unique_ptr<VideoCodecTestStats> stats; + rtc::Event ready; + task_queue_.PostTask([this, &stats, &ready]() mutable { + RTC_DCHECK_RUN_ON(&sequence_checker_); + stats.reset(new VideoCodecTestStatsImpl(stats_)); + ready.Set(); + }); + ready.Wait(rtc::Event::kForever); + return stats; +} + +} // namespace test +} // namespace webrtc
diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.h b/modules/video_coding/codecs/test/video_codec_analyzer.h new file mode 100644 index 0000000..63a864e --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer.h
@@ -0,0 +1,65 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ + +#include <memory> + +#include "absl/types/optional.h" +#include "api/sequence_checker.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/task_queue_for_test.h" + +namespace webrtc { +namespace test { + +// Analyzer measures and collects metrics necessary for evaluation of video +// codec quality and performance. This class is thread-safe. +class VideoCodecAnalyzer { + public: + // An interface that provides reference frames for spatial quality analysis. + class ReferenceVideoSource { + public: + virtual ~ReferenceVideoSource() = default; + + virtual VideoFrame GetFrame(uint32_t timestamp_rtp, + Resolution resolution) = 0; + }; + + VideoCodecAnalyzer(rtc::TaskQueue& task_queue, + ReferenceVideoSource* reference_video_source = nullptr); + + void StartEncode(const VideoFrame& frame); + + void FinishEncode(const EncodedImage& frame); + + void StartDecode(const EncodedImage& frame); + + void FinishDecode(const VideoFrame& frame, int spatial_idx); + + std::unique_ptr<VideoCodecTestStats> GetStats(); + + protected: + rtc::TaskQueue& task_queue_; + ReferenceVideoSource* const reference_video_source_; + VideoCodecTestStatsImpl stats_ RTC_GUARDED_BY(sequence_checker_); + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_
diff --git a/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc b/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc new file mode 100644 index 0000000..3f9de6d --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc
@@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" + +#include "absl/types/optional.h" +#include "api/video/i420_buffer.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "third_party/libyuv/include/libyuv/planar_functions.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::Return; +using ::testing::Values; + +const size_t kTimestamp = 3000; +const size_t kSpatialIdx = 2; + +class MockReferenceVideoSource + : public VideoCodecAnalyzer::ReferenceVideoSource { + public: + MOCK_METHOD(VideoFrame, GetFrame, (uint32_t, Resolution), (override)); +}; + +VideoFrame CreateVideoFrame(uint32_t timestamp_rtp, + uint8_t y = 0, + uint8_t u = 0, + uint8_t v = 0) { + rtc::scoped_refptr<I420Buffer> buffer(I420Buffer::Create(2, 2)); + + libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), + buffer->MutableDataV(), buffer->StrideV(), 0, 0, + buffer->width(), buffer->height(), y, u, v); + + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp, int spatial_idx = 0) { + EncodedImage encoded_image; + encoded_image.SetTimestamp(timestamp_rtp); + encoded_image.SetSpatialIndex(spatial_idx); + return encoded_image; +} +} // namespace + +TEST(VideoCodecAnalyzerTest, EncodeStartedCreatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + EXPECT_EQ(fs[0].rtp_timestamp, kTimestamp); +} + +TEST(VideoCodecAnalyzerTest, EncodeFinishedUpdatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + + EncodedImage encoded_frame = CreateEncodedImage(kTimestamp, kSpatialIdx); + analyzer.FinishEncode(encoded_frame); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(2u, fs.size()); + EXPECT_TRUE(fs[1].encoding_successful); +} + +TEST(VideoCodecAnalyzerTest, DecodeStartedNoFrameStatsCreatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + EXPECT_EQ(fs[0].rtp_timestamp, kTimestamp); +} + +TEST(VideoCodecAnalyzerTest, DecodeStartedFrameStatsExistsReusesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, /*spatial_idx=*/0)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); +} + +TEST(VideoCodecAnalyzerTest, DecodeFinishedUpdatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + VideoFrame decoded_frame = CreateVideoFrame(kTimestamp); + analyzer.FinishDecode(decoded_frame, kSpatialIdx); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + + EXPECT_TRUE(fs[0].decoding_successful); + EXPECT_EQ(static_cast<int>(fs[0].decoded_width), decoded_frame.width()); + EXPECT_EQ(static_cast<int>(fs[0].decoded_height), decoded_frame.height()); +} + +TEST(VideoCodecAnalyzerTest, DecodeFinishedComputesPsnr) { + TaskQueueForTest task_queue; + MockReferenceVideoSource reference_video_source; + VideoCodecAnalyzer analyzer(task_queue, &reference_video_source); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + + EXPECT_CALL(reference_video_source, GetFrame) + .WillOnce(Return(CreateVideoFrame(kTimestamp, /*y=*/0, + /*u=*/0, /*v=*/0))); + + analyzer.FinishDecode( + CreateVideoFrame(kTimestamp, /*value_y=*/1, /*value_u=*/2, /*value_v=*/3), + kSpatialIdx); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + + EXPECT_NEAR(fs[0].psnr_y, 48, 1); + EXPECT_NEAR(fs[0].psnr_u, 42, 1); + EXPECT_NEAR(fs[0].psnr_v, 38, 1); +} + +} // namespace test +} // namespace webrtc
diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc new file mode 100644 index 0000000..bd4c8e0 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_test.cc
@@ -0,0 +1,456 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_codec.h" + +#include <cstddef> +#include <memory> +#include <string> +#include <vector> + +#include "absl/functional/any_invocable.h" +#include "api/test/create_video_codec_tester.h" +#include "api/test/videocodec_test_stats.h" +#include "api/units/data_rate.h" +#include "api/units/frequency.h" +#include "api/video/i420_buffer.h" +#include "api/video/resolution.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "media/base/media_constants.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::Combine; +using ::testing::Values; +using Layer = std::pair<int, int>; + +struct VideoInfo { + std::string name; + Resolution resolution; +}; + +struct CodecInfo { + std::string type; + std::string encoder; + std::string decoder; +}; + +struct EncodingSettings { + ScalabilityMode scalability_mode; + // Spatial layer resolution. + std::map<int, Resolution> resolution; + // Top temporal layer frame rate. + Frequency framerate; + // Bitrate of spatial and temporal layers. + std::map<Layer, DataRate> bitrate; +}; + +struct EncodingTestSettings { + std::string name; + int num_frames = 1; + std::map<int, EncodingSettings> frame_settings; +}; + +struct DecodingTestSettings { + std::string name; +}; + +struct QualityExpectations { + double min_apsnr_y; +}; + +struct EncodeDecodeTestParams { + CodecInfo codec; + VideoInfo video; + VideoCodecTester::EncoderSettings encoder_settings; + VideoCodecTester::DecoderSettings decoder_settings; + EncodingTestSettings encoding_settings; + DecodingTestSettings decoding_settings; + QualityExpectations quality_expectations; +}; + +const EncodingSettings kQvga64Kbps30Fps = { + .scalability_mode = ScalabilityMode::kL1T1, + .resolution = {{0, {.width = 320, .height = 180}}}, + .framerate = Frequency::Hertz(30), + .bitrate = {{Layer(0, 0), DataRate::KilobitsPerSec(64)}}}; + +const EncodingTestSettings kConstantRateQvga64Kbps30Fps = { + .name = "ConstantRateQvga64Kbps30Fps", + .num_frames = 300, + .frame_settings = {{/*frame_num=*/0, kQvga64Kbps30Fps}}}; + +const QualityExpectations kLowQuality = {.min_apsnr_y = 30}; + +const VideoInfo kFourPeople_1280x720_30 = { + .name = "FourPeople_1280x720_30", + .resolution = {.width = 1280, .height = 720}}; + +const CodecInfo kLibvpxVp8 = {.type = "VP8", + .encoder = "libvpx", + .decoder = "libvpx"}; + +const CodecInfo kLibvpxVp9 = {.type = "VP9", + .encoder = "libvpx", + .decoder = "libvpx"}; + +const CodecInfo kOpenH264 = {.type = "H264", + .encoder = "openh264", + .decoder = "ffmpeg"}; + +class TestRawVideoSource : public VideoCodecTester::RawVideoSource { + public: + static constexpr Frequency k90kHz = Frequency::Hertz(90000); + + TestRawVideoSource(std::unique_ptr<FrameReader> frame_reader, + const EncodingTestSettings& test_settings) + : frame_reader_(std::move(frame_reader)), + test_settings_(test_settings), + frame_num_(0), + timestamp_rtp_(0) { + // Ensure settings for the first frame are provided. + RTC_CHECK_GT(test_settings_.frame_settings.size(), 0u); + RTC_CHECK_EQ(test_settings_.frame_settings.begin()->first, 0); + } + + // Pulls next frame. Frame RTP timestamp is set accordingly to + // `EncodingSettings::framerate`. + absl::optional<VideoFrame> PullFrame() override { + if (frame_num_ >= test_settings_.num_frames) { + // End of stream. + return absl::nullopt; + } + + EncodingSettings frame_settings = + std::prev(test_settings_.frame_settings.upper_bound(frame_num_)) + ->second; + + int pulled_frame; + auto buffer = frame_reader_->PullFrame( + &pulled_frame, frame_settings.resolution.rbegin()->second, + {.num = 30, .den = static_cast<int>(frame_settings.framerate.hertz())}); + RTC_CHECK(buffer) << "Cannot pull frame " << frame_num_; + + auto frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp_) + .build(); + + pulled_frames_[timestamp_rtp_] = pulled_frame; + timestamp_rtp_ += k90kHz / frame_settings.framerate; + ++frame_num_; + + return frame; + } + + // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and + // returns. Frame with the given `timestamp_rtp` is expected to be pulled + // before. + VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { + RTC_CHECK(pulled_frames_.find(timestamp_rtp) != pulled_frames_.end()) + << "Frame with RTP timestamp " << timestamp_rtp + << " was not pulled before"; + auto buffer = + frame_reader_->ReadFrame(pulled_frames_[timestamp_rtp], resolution); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); + } + + protected: + std::unique_ptr<FrameReader> frame_reader_; + const EncodingTestSettings& test_settings_; + int frame_num_; + uint32_t timestamp_rtp_; + std::map<uint32_t, int> pulled_frames_; +}; + +class TestEncoder : public VideoCodecTester::Encoder, + public EncodedImageCallback { + public: + TestEncoder(std::unique_ptr<VideoEncoder> encoder, + const CodecInfo& codec_info, + const std::map<int, EncodingSettings>& frame_settings) + : encoder_(std::move(encoder)), + codec_info_(codec_info), + frame_settings_(frame_settings), + frame_num_(0) { + // Ensure settings for the first frame is provided. + RTC_CHECK_GT(frame_settings_.size(), 0u); + RTC_CHECK_EQ(frame_settings_.begin()->first, 0); + + encoder_->RegisterEncodeCompleteCallback(this); + } + + void Encode(const VideoFrame& frame, EncodeCallback callback) override { + callbacks_[frame.timestamp()] = std::move(callback); + + if (auto fs = frame_settings_.find(frame_num_); + fs != frame_settings_.end()) { + if (fs == frame_settings_.begin() || + ConfigChanged(fs->second, std::prev(fs)->second)) { + Configure(fs->second); + } + if (fs == frame_settings_.begin() || + RateChanged(fs->second, std::prev(fs)->second)) { + SetRates(fs->second); + } + } + + int result = encoder_->Encode(frame, nullptr); + RTC_CHECK_EQ(result, WEBRTC_VIDEO_CODEC_OK); + ++frame_num_; + } + + protected: + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override { + auto cb = callbacks_.find(encoded_image.Timestamp()); + RTC_CHECK(cb != callbacks_.end()); + cb->second(encoded_image); + + callbacks_.erase(callbacks_.begin(), cb); + return Result(Result::Error::OK); + } + + void Configure(const EncodingSettings& es) { + VideoCodec vc; + const Resolution& resolution = es.resolution.rbegin()->second; + vc.width = resolution.width; + vc.height = resolution.height; + const DataRate& bitrate = es.bitrate.rbegin()->second; + vc.startBitrate = bitrate.kbps(); + vc.maxBitrate = bitrate.kbps(); + vc.minBitrate = 0; + vc.maxFramerate = static_cast<uint32_t>(es.framerate.hertz()); + vc.active = true; + vc.qpMax = 0; + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetFrameDropEnabled(true); + + vc.codecType = PayloadStringToCodecType(codec_info_.type); + if (vc.codecType == kVideoCodecVP8) { + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + } else if (vc.codecType == kVideoCodecVP9) { + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + } else if (vc.codecType == kVideoCodecH264) { + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + } + + VideoEncoder::Settings ves( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, + /*max_payload_size=*/1440); + + int result = encoder_->InitEncode(&vc, ves); + RTC_CHECK_EQ(result, WEBRTC_VIDEO_CODEC_OK); + } + + void SetRates(const EncodingSettings& es) { + VideoEncoder::RateControlParameters rc; + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + RTC_CHECK(es.bitrate.find(Layer(sidx, tidx)) != es.bitrate.end()) + << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; + rc.bitrate.SetBitrate(sidx, tidx, + es.bitrate.at(Layer(sidx, tidx)).bps()); + } + } + + rc.framerate_fps = es.framerate.millihertz() / 1000.0; + encoder_->SetRates(rc); + } + + bool ConfigChanged(const EncodingSettings& es, + const EncodingSettings& prev_es) const { + return es.scalability_mode != prev_es.scalability_mode || + es.resolution != prev_es.resolution; + } + + bool RateChanged(const EncodingSettings& es, + const EncodingSettings& prev_es) const { + return es.bitrate != prev_es.bitrate || es.framerate != prev_es.framerate; + } + + std::unique_ptr<VideoEncoder> encoder_; + const CodecInfo& codec_info_; + const std::map<int, EncodingSettings>& frame_settings_; + int frame_num_; + std::map<uint32_t, EncodeCallback> callbacks_; +}; + +class TestDecoder : public VideoCodecTester::Decoder, + public DecodedImageCallback { + public: + TestDecoder(std::unique_ptr<VideoDecoder> decoder, + const CodecInfo& codec_info) + : decoder_(std::move(decoder)), codec_info_(codec_info), frame_num_(0) { + decoder_->RegisterDecodeCompleteCallback(this); + } + void Decode(const EncodedImage& frame, DecodeCallback callback) override { + callbacks_[frame.Timestamp()] = std::move(callback); + + if (frame_num_ == 0) { + Configure(); + } + + decoder_->Decode(frame, /*missing_frames=*/false, + /*render_time_ms=*/0); + ++frame_num_; + } + + void Configure() { + VideoDecoder::Settings ds; + ds.set_codec_type(PayloadStringToCodecType(codec_info_.type)); + ds.set_number_of_cores(1); + + bool result = decoder_->Configure(ds); + RTC_CHECK(result); + } + + protected: + int Decoded(VideoFrame& decoded_frame) override { + auto cb = callbacks_.find(decoded_frame.timestamp()); + RTC_CHECK(cb != callbacks_.end()); + cb->second(decoded_frame); + + callbacks_.erase(callbacks_.begin(), cb); + return WEBRTC_VIDEO_CODEC_OK; + } + + std::unique_ptr<VideoDecoder> decoder_; + const CodecInfo& codec_info_; + int frame_num_; + std::map<uint32_t, DecodeCallback> callbacks_; +}; + +std::unique_ptr<VideoCodecTester::Encoder> CreateEncoder( + const CodecInfo& codec_info, + const std::map<int, EncodingSettings>& frame_settings) { + auto factory = CreateBuiltinVideoEncoderFactory(); + auto encoder = factory->CreateVideoEncoder(SdpVideoFormat(codec_info.type)); + return std::make_unique<TestEncoder>(std::move(encoder), codec_info, + frame_settings); +} + +std::unique_ptr<VideoCodecTester::Decoder> CreateDecoder( + const CodecInfo& codec_info) { + auto factory = CreateBuiltinVideoDecoderFactory(); + auto decoder = factory->CreateVideoDecoder(SdpVideoFormat(codec_info.type)); + return std::make_unique<TestDecoder>(std::move(decoder), codec_info); +} + +} // namespace + +class EncodeDecodeTest + : public ::testing::TestWithParam<EncodeDecodeTestParams> { + public: + EncodeDecodeTest() : test_params_(GetParam()) {} + + void SetUp() override { + std::unique_ptr<FrameReader> frame_reader = + CreateYuvFrameReader(ResourcePath(test_params_.video.name, "yuv"), + test_params_.video.resolution, + YuvFrameReaderImpl::RepeatMode::kPingPong); + video_source_ = std::make_unique<TestRawVideoSource>( + std::move(frame_reader), test_params_.encoding_settings); + + encoder_ = CreateEncoder(test_params_.codec, + test_params_.encoding_settings.frame_settings); + decoder_ = CreateDecoder(test_params_.codec); + + tester_ = CreateVideoCodecTester(); + } + + static std::string TestParametersToStr( + const ::testing::TestParamInfo<EncodeDecodeTest::ParamType>& info) { + return std::string(info.param.encoding_settings.name + + info.param.codec.type + info.param.codec.encoder + + info.param.codec.decoder); + } + + protected: + EncodeDecodeTestParams test_params_; + std::unique_ptr<TestRawVideoSource> video_source_; + std::unique_ptr<VideoCodecTester::Encoder> encoder_; + std::unique_ptr<VideoCodecTester::Decoder> decoder_; + std::unique_ptr<VideoCodecTester> tester_; +}; + +TEST_P(EncodeDecodeTest, DISABLED_TestEncodeDecode) { + std::unique_ptr<VideoCodecTestStats> stats = tester_->RunEncodeDecodeTest( + std::move(video_source_), std::move(encoder_), std::move(decoder_), + test_params_.encoder_settings, test_params_.decoder_settings); + + const auto& frame_settings = test_params_.encoding_settings.frame_settings; + for (auto fs = frame_settings.begin(); fs != frame_settings.end(); ++fs) { + int first_frame = fs->first; + int last_frame = std::next(fs) != frame_settings.end() + ? std::next(fs)->first - 1 + : test_params_.encoding_settings.num_frames - 1; + + const EncodingSettings& encoding_settings = fs->second; + auto metrics = stats->CalcVideoStatistic( + first_frame, last_frame, encoding_settings.bitrate.rbegin()->second, + encoding_settings.framerate); + + EXPECT_GE(metrics.avg_psnr_y, + test_params_.quality_expectations.min_apsnr_y); + } +} + +std::list<EncodeDecodeTestParams> ConstantRateTestParameters() { + std::list<EncodeDecodeTestParams> test_params; + std::vector<CodecInfo> codecs = {kLibvpxVp8}; + std::vector<VideoInfo> videos = {kFourPeople_1280x720_30}; + std::vector<std::pair<EncodingTestSettings, QualityExpectations>> + encoding_settings = {{kConstantRateQvga64Kbps30Fps, kLowQuality}}; + for (const CodecInfo& codec : codecs) { + for (const VideoInfo& video : videos) { + for (const auto& es : encoding_settings) { + EncodeDecodeTestParams p; + p.codec = codec; + p.video = video; + p.encoding_settings = es.first; + p.quality_expectations = es.second; + test_params.push_back(p); + } + } + } + return test_params; +} + +INSTANTIATE_TEST_SUITE_P(ConstantRate, + EncodeDecodeTest, + ::testing::ValuesIn(ConstantRateTestParameters()), + EncodeDecodeTest::TestParametersToStr); +} // namespace test + +} // namespace webrtc
diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.cc b/modules/video_coding/codecs/test/video_codec_tester_impl.cc new file mode 100644 index 0000000..3000c1a --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl.cc
@@ -0,0 +1,325 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +#include <map> +#include <memory> +#include <utility> + +#include "api/task_queue/default_task_queue_factory.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" +#include "rtc_base/event.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/sleep.h" + +namespace webrtc { +namespace test { + +namespace { +using RawVideoSource = VideoCodecTester::RawVideoSource; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using Decoder = VideoCodecTester::Decoder; +using Encoder = VideoCodecTester::Encoder; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +// A thread-safe wrapper for video source to be shared with the quality analyzer +// that reads reference frames from a separate thread. +class SyncRawVideoSource : public VideoCodecAnalyzer::ReferenceVideoSource { + public: + explicit SyncRawVideoSource(std::unique_ptr<RawVideoSource> video_source) + : video_source_(std::move(video_source)) {} + + absl::optional<VideoFrame> PullFrame() { + MutexLock lock(&mutex_); + return video_source_->PullFrame(); + } + + VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { + MutexLock lock(&mutex_); + return video_source_->GetFrame(timestamp_rtp, resolution); + } + + protected: + std::unique_ptr<RawVideoSource> video_source_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +// Pacer calculates delay necessary to keep frame encode or decode call spaced +// from the previous calls by the pacing time. `Delay` is expected to be called +// as close as possible to posting frame encode or decode task. This class is +// not thread safe. +class Pacer { + public: + explicit Pacer(PacingSettings settings) + : settings_(settings), delay_(TimeDelta::Zero()) {} + TimeDelta Delay(Timestamp beat) { + if (settings_.mode == PacingMode::kNoPacing) { + return TimeDelta::Zero(); + } + + Timestamp now = Timestamp::Micros(rtc::TimeMicros()); + if (prev_time_.has_value()) { + delay_ += PacingTime(beat); + delay_ -= (now - *prev_time_); + if (delay_.ns() < 0) { + delay_ = TimeDelta::Zero(); + } + } + + prev_beat_ = beat; + prev_time_ = now; + return delay_; + } + + private: + TimeDelta PacingTime(Timestamp beat) { + if (settings_.mode == PacingMode::kRealTime) { + return beat - *prev_beat_; + } + RTC_CHECK_EQ(PacingMode::kConstantRate, settings_.mode); + return 1 / settings_.constant_rate; + } + + PacingSettings settings_; + absl::optional<Timestamp> prev_beat_; + absl::optional<Timestamp> prev_time_; + TimeDelta delay_; +}; + +// Task queue that keeps the number of queued tasks below a certain limit. If +// the limit is reached, posting of a next task is blocked until execution of a +// previously posted task starts. This class is not thread-safe. +class LimitedTaskQueue { + public: + // The codec tester reads frames from video source in the main thread. + // Encoding and decoding are done in separate threads. If encoding or + // decoding is slow, the reading may go far ahead and may buffer too many + // frames in memory. To prevent this we limit the encoding/decoding queue + // size. When the queue is full, the main thread and, hence, reading frames + // from video source is blocked until a previously posted encoding/decoding + // task starts. + static constexpr int kMaxTaskQueueSize = 3; + + explicit LimitedTaskQueue(rtc::TaskQueue& task_queue) + : task_queue_(task_queue), queue_size_(0) {} + + void PostDelayedTask(absl::AnyInvocable<void() &&> task, TimeDelta delay) { + ++queue_size_; + task_queue_.PostDelayedTask( + [this, task = std::move(task)]() mutable { + std::move(task)(); + --queue_size_; + task_executed_.Set(); + }, + delay); + + task_executed_.Reset(); + if (queue_size_ > kMaxTaskQueueSize) { + task_executed_.Wait(rtc::Event::kForever); + } + RTC_CHECK(queue_size_ <= kMaxTaskQueueSize); + } + + void WaitForPreviouslyPostedTasks() { + while (queue_size_ > 0) { + task_executed_.Wait(rtc::Event::kForever); + task_executed_.Reset(); + } + } + + rtc::TaskQueue& task_queue_; + std::atomic_int queue_size_; + rtc::Event task_executed_; +}; + +class TesterDecoder { + public: + TesterDecoder(std::unique_ptr<Decoder> decoder, + VideoCodecAnalyzer* analyzer, + const DecoderSettings& settings, + rtc::TaskQueue& task_queue) + : decoder_(std::move(decoder)), + analyzer_(analyzer), + settings_(settings), + pacer_(settings.pacing), + task_queue_(task_queue) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + } + + void Decode(const EncodedImage& frame) { + Timestamp timestamp = Timestamp::Micros((frame.Timestamp() / k90kHz).us()); + + task_queue_.PostDelayedTask( + [this, frame] { + analyzer_->StartDecode(frame); + decoder_->Decode(frame, [this](const VideoFrame& decoded_frame) { + this->analyzer_->FinishDecode(decoded_frame, /*spatial_idx=*/0); + }); + }, + pacer_.Delay(timestamp)); + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + protected: + std::unique_ptr<Decoder> decoder_; + VideoCodecAnalyzer* const analyzer_; + const DecoderSettings& settings_; + Pacer pacer_; + LimitedTaskQueue task_queue_; +}; + +class TesterEncoder { + public: + TesterEncoder(std::unique_ptr<Encoder> encoder, + TesterDecoder* decoder, + VideoCodecAnalyzer* analyzer, + const EncoderSettings& settings, + rtc::TaskQueue& task_queue) + : encoder_(std::move(encoder)), + decoder_(decoder), + analyzer_(analyzer), + settings_(settings), + pacer_(settings.pacing), + task_queue_(task_queue) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + } + + void Encode(const VideoFrame& frame) { + Timestamp timestamp = Timestamp::Micros((frame.timestamp() / k90kHz).us()); + + task_queue_.PostDelayedTask( + [this, frame] { + analyzer_->StartEncode(frame); + encoder_->Encode(frame, [this](const EncodedImage& encoded_frame) { + this->analyzer_->FinishEncode(encoded_frame); + if (decoder_ != nullptr) { + this->decoder_->Decode(encoded_frame); + } + }); + }, + pacer_.Delay(timestamp)); + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + protected: + std::unique_ptr<Encoder> encoder_; + TesterDecoder* const decoder_; + VideoCodecAnalyzer* const analyzer_; + const EncoderSettings& settings_; + Pacer pacer_; + LimitedTaskQueue task_queue_; +}; + +} // namespace + +VideoCodecTesterImpl::VideoCodecTesterImpl() + : VideoCodecTesterImpl(/*task_queue_factory=*/nullptr) {} + +VideoCodecTesterImpl::VideoCodecTesterImpl(TaskQueueFactory* task_queue_factory) + : task_queue_factory_(task_queue_factory) { + if (task_queue_factory_ == nullptr) { + owned_task_queue_factory_ = CreateDefaultTaskQueueFactory(); + task_queue_factory_ = owned_task_queue_factory_.get(); + } +} + +std::unique_ptr<VideoCodecTestStats> VideoCodecTesterImpl::RunDecodeTest( + std::unique_ptr<CodedVideoSource> video_source, + std::unique_ptr<Decoder> decoder, + const DecoderSettings& decoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue decoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Decoder", TaskQueueFactory::Priority::NORMAL)); + + VideoCodecAnalyzer perf_analyzer(analyser_task_queue); + TesterDecoder tester_decoder(std::move(decoder), &perf_analyzer, + decoder_settings, decoder_task_queue); + + while (auto frame = video_source->PullFrame()) { + tester_decoder.Decode(*frame); + } + + tester_decoder.Flush(); + + return perf_analyzer.GetStats(); +} + +std::unique_ptr<VideoCodecTestStats> VideoCodecTesterImpl::RunEncodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + const EncoderSettings& encoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue encoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Encoder", TaskQueueFactory::Priority::NORMAL)); + + SyncRawVideoSource sync_source(std::move(video_source)); + VideoCodecAnalyzer perf_analyzer(analyser_task_queue); + TesterEncoder tester_encoder(std::move(encoder), /*decoder=*/nullptr, + &perf_analyzer, encoder_settings, + encoder_task_queue); + + while (auto frame = sync_source.PullFrame()) { + tester_encoder.Encode(*frame); + } + + tester_encoder.Flush(); + + return perf_analyzer.GetStats(); +} + +std::unique_ptr<VideoCodecTestStats> VideoCodecTesterImpl::RunEncodeDecodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + std::unique_ptr<Decoder> decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue decoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Decoder", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue encoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Encoder", TaskQueueFactory::Priority::NORMAL)); + + SyncRawVideoSource sync_source(std::move(video_source)); + VideoCodecAnalyzer perf_analyzer(analyser_task_queue, &sync_source); + TesterDecoder tester_decoder(std::move(decoder), &perf_analyzer, + decoder_settings, decoder_task_queue); + TesterEncoder tester_encoder(std::move(encoder), &tester_decoder, + &perf_analyzer, encoder_settings, + encoder_task_queue); + + while (auto frame = sync_source.PullFrame()) { + tester_encoder.Encode(*frame); + } + + tester_encoder.Flush(); + tester_decoder.Flush(); + + return perf_analyzer.GetStats(); +} + +} // namespace test +} // namespace webrtc
diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.h b/modules/video_coding/codecs/test/video_codec_tester_impl.h new file mode 100644 index 0000000..b64adeb --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl.h
@@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ + +#include <memory> + +#include "api/task_queue/task_queue_factory.h" +#include "api/test/video_codec_tester.h" + +namespace webrtc { +namespace test { + +// A stateless implementation of `VideoCodecTester`. This class is thread safe. +class VideoCodecTesterImpl : public VideoCodecTester { + public: + VideoCodecTesterImpl(); + explicit VideoCodecTesterImpl(TaskQueueFactory* task_queue_factory); + + std::unique_ptr<VideoCodecTestStats> RunDecodeTest( + std::unique_ptr<CodedVideoSource> video_source, + std::unique_ptr<Decoder> decoder, + const DecoderSettings& decoder_settings) override; + + std::unique_ptr<VideoCodecTestStats> RunEncodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + const EncoderSettings& encoder_settings) override; + + std::unique_ptr<VideoCodecTestStats> RunEncodeDecodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + std::unique_ptr<Decoder> decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) override; + + protected: + std::unique_ptr<TaskQueueFactory> owned_task_queue_factory_; + TaskQueueFactory* task_queue_factory_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_
diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc new file mode 100644 index 0000000..29fb006 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc
@@ -0,0 +1,259 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +#include <memory> +#include <tuple> +#include <utility> +#include <vector> + +#include "api/task_queue/task_queue_factory.h" +#include "api/task_queue/test/mock_task_queue_base.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/gunit.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::_; +using ::testing::Invoke; +using ::testing::InvokeWithoutArgs; +using ::testing::Return; + +using Decoder = VideoCodecTester::Decoder; +using Encoder = VideoCodecTester::Encoder; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using RawVideoSource = VideoCodecTester::RawVideoSource; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +VideoFrame CreateVideoFrame(uint32_t timestamp_rtp) { + rtc::scoped_refptr<I420Buffer> buffer(I420Buffer::Create(2, 2)); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) { + EncodedImage encoded_image; + encoded_image.SetTimestamp(timestamp_rtp); + return encoded_image; +} + +class MockRawVideoSource : public RawVideoSource { + public: + MOCK_METHOD(absl::optional<VideoFrame>, PullFrame, (), (override)); + MOCK_METHOD(VideoFrame, + GetFrame, + (uint32_t timestamp_rtp, Resolution), + (override)); +}; + +class MockCodedVideoSource : public CodedVideoSource { + public: + MOCK_METHOD(absl::optional<EncodedImage>, PullFrame, (), (override)); +}; + +class MockDecoder : public Decoder { + public: + MOCK_METHOD(void, + Decode, + (const EncodedImage& frame, DecodeCallback callback), + (override)); +}; + +class MockEncoder : public Encoder { + public: + MOCK_METHOD(void, + Encode, + (const VideoFrame& frame, EncodeCallback callback), + (override)); +}; + +class MockTaskQueueFactory : public TaskQueueFactory { + public: + explicit MockTaskQueueFactory(TaskQueueBase& task_queue) + : task_queue_(task_queue) {} + + std::unique_ptr<TaskQueueBase, TaskQueueDeleter> CreateTaskQueue( + absl::string_view name, + Priority priority) const override { + return std::unique_ptr<TaskQueueBase, TaskQueueDeleter>(&task_queue_); + } + + protected: + TaskQueueBase& task_queue_; +}; +} // namespace + +class VideoCodecTesterImplPacingTest + : public ::testing::TestWithParam<std::tuple<PacingSettings, + std::vector<int>, + std::vector<int>, + std::vector<int>>> { + public: + VideoCodecTesterImplPacingTest() + : pacing_settings_(std::get<0>(GetParam())), + frame_timestamp_ms_(std::get<1>(GetParam())), + frame_capture_delay_ms_(std::get<2>(GetParam())), + expected_frame_start_ms_(std::get<3>(GetParam())), + num_frames_(frame_timestamp_ms_.size()), + task_queue_factory_(task_queue_) {} + + void SetUp() override { + ON_CALL(task_queue_, PostTask) + .WillByDefault(Invoke( + [](absl::AnyInvocable<void() &&> task) { std::move(task)(); })); + + ON_CALL(task_queue_, PostDelayedTask) + .WillByDefault( + Invoke([&](absl::AnyInvocable<void() &&> task, TimeDelta delay) { + clock_.AdvanceTime(delay); + std::move(task)(); + })); + } + + protected: + PacingSettings pacing_settings_; + std::vector<int> frame_timestamp_ms_; + std::vector<int> frame_capture_delay_ms_; + std::vector<int> expected_frame_start_ms_; + size_t num_frames_; + + rtc::ScopedFakeClock clock_; + MockTaskQueueBase task_queue_; + MockTaskQueueFactory task_queue_factory_; +}; + +TEST_P(VideoCodecTesterImplPacingTest, PaceEncode) { + auto video_source = std::make_unique<MockRawVideoSource>(); + + size_t frame_num = 0; + EXPECT_CALL(*video_source, PullFrame).WillRepeatedly(Invoke([&]() mutable { + if (frame_num >= num_frames_) { + return absl::optional<VideoFrame>(); + } + clock_.AdvanceTime(TimeDelta::Millis(frame_capture_delay_ms_[frame_num])); + + uint32_t timestamp_rtp = frame_timestamp_ms_[frame_num] * k90kHz.hertz() / + rtc::kNumMillisecsPerSec; + ++frame_num; + return absl::optional<VideoFrame>(CreateVideoFrame(timestamp_rtp)); + })); + + auto encoder = std::make_unique<MockEncoder>(); + EncoderSettings encoder_settings; + encoder_settings.pacing = pacing_settings_; + + VideoCodecTesterImpl tester(&task_queue_factory_); + auto fs = tester + .RunEncodeTest(std::move(video_source), std::move(encoder), + encoder_settings) + ->GetFrameStatistics(); + ASSERT_EQ(fs.size(), num_frames_); + + for (size_t i = 0; i < fs.size(); ++i) { + int encode_start_ms = (fs[i].encode_start_ns - fs[0].encode_start_ns) / + rtc::kNumNanosecsPerMillisec; + EXPECT_NEAR(encode_start_ms, expected_frame_start_ms_[i], 10); + } +} + +TEST_P(VideoCodecTesterImplPacingTest, PaceDecode) { + auto video_source = std::make_unique<MockCodedVideoSource>(); + + size_t frame_num = 0; + EXPECT_CALL(*video_source, PullFrame).WillRepeatedly(Invoke([&]() mutable { + if (frame_num >= num_frames_) { + return absl::optional<EncodedImage>(); + } + clock_.AdvanceTime(TimeDelta::Millis(frame_capture_delay_ms_[frame_num])); + + uint32_t timestamp_rtp = frame_timestamp_ms_[frame_num] * k90kHz.hertz() / + rtc::kNumMillisecsPerSec; + ++frame_num; + return absl::optional<EncodedImage>(CreateEncodedImage(timestamp_rtp)); + })); + + auto decoder = std::make_unique<MockDecoder>(); + DecoderSettings decoder_settings; + decoder_settings.pacing = pacing_settings_; + + VideoCodecTesterImpl tester(&task_queue_factory_); + auto fs = tester + .RunDecodeTest(std::move(video_source), std::move(decoder), + decoder_settings) + ->GetFrameStatistics(); + ASSERT_EQ(fs.size(), num_frames_); + + for (size_t i = 0; i < fs.size(); ++i) { + int decode_start_ms = (fs[i].decode_start_ns - fs[0].decode_start_ns) / + rtc::kNumNanosecsPerMillisec; + EXPECT_NEAR(decode_start_ms, expected_frame_start_ms_[i], 10); + } +} + +INSTANTIATE_TEST_SUITE_P( + All, + VideoCodecTesterImplPacingTest, + ::testing::ValuesIn( + {std::make_tuple(PacingSettings({.mode = PacingMode::kNoPacing}), + /*frame_timestamp_ms=*/std::vector<int>{0, 100}, + /*frame_capture_delay_ms=*/std::vector<int>{0, 0}, + /*expected_frame_start_ms=*/std::vector<int>{0, 0}), + // Pace with rate equal to the source frame rate. Frames are captured + // instantly. Verify that frames are paced with the source frame rate. + std::make_tuple(PacingSettings({.mode = PacingMode::kRealTime}), + /*frame_timestamp_ms=*/std::vector<int>{0, 100}, + /*frame_capture_delay_ms=*/std::vector<int>{0, 0}, + /*expected_frame_start_ms=*/std::vector<int>{0, 100}), + // Pace with rate equal to the source frame rate. Frame capture is + // delayed by more than pacing time. Verify that no extra delay is + // added. + std::make_tuple(PacingSettings({.mode = PacingMode::kRealTime}), + /*frame_timestamp_ms=*/std::vector<int>{0, 100}, + /*frame_capture_delay_ms=*/std::vector<int>{0, 200}, + /*expected_frame_start_ms=*/std::vector<int>{0, 200}), + // Pace with constant rate less then source frame rate. Frames are + // captured instantly. Verify that frames are paced with the requested + // constant rate. + std::make_tuple( + PacingSettings({.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}), + /*frame_timestamp_ms=*/std::vector<int>{0, 100}, + /*frame_capture_delay_ms=*/std::vector<int>{0, 0}, + /*expected_frame_start_ms=*/std::vector<int>{0, 50}), + // Pace with constant rate less then source frame rate. Frame capture + // is delayed by more than the pacing time. Verify that no extra delay + // is added. + std::make_tuple( + PacingSettings({.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}), + /*frame_timestamp_ms=*/std::vector<int>{0, 100}, + /*frame_capture_delay_ms=*/std::vector<int>{0, 200}, + /*expected_frame_start_ms=*/std::vector<int>{0, 200})})); +} // namespace test +} // namespace webrtc
diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc index efb7502..390348b 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc
@@ -58,7 +58,20 @@ return GetFrame(rtp_timestamp_to_frame_num_[layer_idx][timestamp], layer_idx); } -std::vector<FrameStatistics> VideoCodecTestStatsImpl::GetFrameStatistics() { +FrameStatistics* VideoCodecTestStatsImpl::GetOrAddFrame(size_t timestamp_rtp, + size_t spatial_idx) { + if (rtp_timestamp_to_frame_num_[spatial_idx].count(timestamp_rtp) > 0) { + return GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + } + + size_t frame_num = layer_stats_[spatial_idx].size(); + AddFrame(FrameStatistics(frame_num, timestamp_rtp, spatial_idx)); + + return GetFrameWithTimestamp(timestamp_rtp, spatial_idx); +} + +std::vector<FrameStatistics> VideoCodecTestStatsImpl::GetFrameStatistics() + const { size_t capacity = 0; for (const auto& layer_stat : layer_stats_) { capacity += layer_stat.second.size(); @@ -92,7 +105,8 @@ for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers; ++temporal_idx) { VideoStatistics layer_stat = SliceAndCalcVideoStatistic( - first_frame_num, last_frame_num, spatial_idx, temporal_idx, false); + first_frame_num, last_frame_num, spatial_idx, temporal_idx, false, + /*target_bitrate=*/absl::nullopt, /*target_framerate=*/absl::nullopt); layer_stats.push_back(layer_stat); } } @@ -110,9 +124,24 @@ RTC_CHECK_GT(num_spatial_layers, 0); RTC_CHECK_GT(num_temporal_layers, 0); - return SliceAndCalcVideoStatistic(first_frame_num, last_frame_num, - num_spatial_layers - 1, - num_temporal_layers - 1, true); + return SliceAndCalcVideoStatistic( + first_frame_num, last_frame_num, num_spatial_layers - 1, + num_temporal_layers - 1, true, /*target_bitrate=*/absl::nullopt, + /*target_framerate=*/absl::nullopt); +} + +VideoStatistics VideoCodecTestStatsImpl::CalcVideoStatistic( + size_t first_frame_num, + size_t last_frame_num, + DataRate target_bitrate, + Frequency target_framerate) { + size_t num_spatial_layers = 0; + size_t num_temporal_layers = 0; + GetNumberOfEncodedLayers(first_frame_num, last_frame_num, &num_spatial_layers, + &num_temporal_layers); + return SliceAndCalcVideoStatistic( + first_frame_num, last_frame_num, num_spatial_layers - 1, + num_temporal_layers - 1, true, target_bitrate, target_framerate); } size_t VideoCodecTestStatsImpl::Size(size_t spatial_idx) { @@ -175,7 +204,9 @@ size_t last_frame_num, size_t spatial_idx, size_t temporal_idx, - bool aggregate_independent_layers) { + bool aggregate_independent_layers, + absl::optional<DataRate> target_bitrate, + absl::optional<Frequency> target_framerate) { VideoStatistics video_stat; float buffer_level_bits = 0.0f; @@ -200,8 +231,11 @@ FrameStatistics last_successfully_decoded_frame(0, 0, 0); const size_t target_bitrate_kbps = - CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, spatial_idx, - temporal_idx, aggregate_independent_layers); + target_bitrate.has_value() + ? target_bitrate->kbps() + : CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, + spatial_idx, temporal_idx, + aggregate_independent_layers); const size_t target_bitrate_bps = 1000 * target_bitrate_kbps; RTC_CHECK_GT(target_bitrate_kbps, 0); // We divide by `target_bitrate_kbps`. @@ -303,7 +337,9 @@ GetFrame(first_frame_num, spatial_idx)->rtp_timestamp; RTC_CHECK_GT(timestamp_delta, 0); const float input_framerate_fps = - 1.0 * kVideoPayloadTypeFrequency / timestamp_delta; + target_framerate.has_value() + ? target_framerate->millihertz() / 1000.0 + : 1.0 * kVideoPayloadTypeFrequency / timestamp_delta; RTC_CHECK_GT(input_framerate_fps, 0); const float duration_sec = num_frames / input_framerate_fps;
diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl.h b/modules/video_coding/codecs/test/videocodec_test_stats_impl.h index 61850d3..1a7980a 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl.h +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl.h
@@ -35,8 +35,12 @@ FrameStatistics* GetFrame(size_t frame_number, size_t spatial_idx); FrameStatistics* GetFrameWithTimestamp(size_t timestamp, size_t spatial_idx); + // Creates FrameStatisticts if it doesn't exists and/or returns + // created/existing FrameStatisticts. + FrameStatistics* GetOrAddFrame(size_t timestamp_rtp, size_t spatial_idx); + // Implements VideoCodecTestStats. - std::vector<FrameStatistics> GetFrameStatistics() override; + std::vector<FrameStatistics> GetFrameStatistics() const override; std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic( size_t first_frame_num, size_t last_frame_num) override; @@ -44,6 +48,11 @@ VideoStatistics SliceAndCalcAggregatedVideoStatistic(size_t first_frame_num, size_t last_frame_num); + VideoStatistics CalcVideoStatistic(size_t first_frame, + size_t last_frame, + DataRate target_bitrate, + Frequency target_framerate) override; + size_t Size(size_t spatial_idx); void Clear(); @@ -65,7 +74,9 @@ size_t last_frame_num, size_t spatial_idx, size_t temporal_idx, - bool aggregate_independent_layers); + bool aggregate_independent_layers, + absl::optional<DataRate> target_bitrate, + absl::optional<Frequency> target_framerate); void GetNumberOfEncodedLayers(size_t first_frame_num, size_t last_frame_num,
diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc b/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc index 6477b6a..89e7d2e 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc
@@ -38,6 +38,21 @@ EXPECT_EQ(kTimestamp, frame_stat->rtp_timestamp); } +TEST(StatsTest, GetOrAddFrame_noFrame_createsNewFrameStat) { + VideoCodecTestStatsImpl stats; + stats.GetOrAddFrame(kTimestamp, 0); + FrameStatistics* frame_stat = stats.GetFrameWithTimestamp(kTimestamp, 0); + EXPECT_EQ(kTimestamp, frame_stat->rtp_timestamp); +} + +TEST(StatsTest, GetOrAddFrame_frameExists_returnsExistingFrameStat) { + VideoCodecTestStatsImpl stats; + stats.AddFrame(FrameStatistics(0, kTimestamp, 0)); + FrameStatistics* frame_stat1 = stats.GetFrameWithTimestamp(kTimestamp, 0); + FrameStatistics* frame_stat2 = stats.GetOrAddFrame(kTimestamp, 0); + EXPECT_EQ(frame_stat1, frame_stat2); +} + TEST(StatsTest, AddAndGetFrames) { VideoCodecTestStatsImpl stats; const size_t kNumFrames = 1000;