Reland "Added an encode/decode test parameterizable via command line"
This is a reland of commit 496893e89e5bc8139e50befcb1a26eadbd829b0d
Original change's description:
> Added an encode/decode test parameterizable via command line
>
> This enables testing different settings without updating code and rebuilding the test binary. Example of command:
>
> video_codec_perf_tests --gtest_also_run_disabled_tests --gtest_filter=*EncodeDecode --encoder=libaom-av1 --decoder=dav1d --scalability_mode=L1T3 --bitrate_kbps=100,200,300 --framerate_fps=30 --write_csv
>
> Also added writing per-frame stats to a CSV. It is more convenient to work with CSV than to parse metrics proto.
>
> Bug: webrtc:14852
> Change-Id: I1b3970f7ffa88c016133197aff585de5bc4e35c6
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/327600
> Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#41179}
Bug: webrtc:14852
Change-Id: Iccb9af8bf6a6c37704bc58b6e57238b55761b079
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/327781
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41194}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index c75b433..d9e614f 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -1005,7 +1005,9 @@
"../../api/video:resolution",
"../../api/video_codecs:builtin_video_decoder_factory",
"../../api/video_codecs:builtin_video_encoder_factory",
+ "../../modules/video_coding/svc:scalability_mode_util",
"../../rtc_base:logging",
+ "../../rtc_base:stringutils",
"../../test:fileutils",
"../../test:test_flags",
"../../test:test_main",
diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc
index 08961a3..60c2fcb 100644
--- a/modules/video_coding/codecs/test/video_codec_test.cc
+++ b/modules/video_coding/codecs/test/video_codec_test.cc
@@ -23,19 +23,47 @@
#if defined(WEBRTC_ANDROID)
#include "modules/video_coding/codecs/test/android_codec_factory_helper.h"
#endif
+#include "modules/video_coding/svc/scalability_mode_util.h"
#include "rtc_base/logging.h"
+#include "rtc_base/strings/string_builder.h"
#include "test/gtest.h"
#include "test/test_flags.h"
#include "test/testsupport/file_utils.h"
#include "test/video_codec_tester.h"
+ABSL_FLAG(std::string,
+ video_name,
+ "FourPeople_1280x720_30",
+ "Name of input video sequence.");
+ABSL_FLAG(std::string,
+ encoder,
+ "libaom-av1",
+ "Encoder: libaom-av1, libvpx-vp9, libvpx-vp8, openh264, hw-vp8, "
+ "hw-vp9, hw-av1, hw-h264, hw-h265");
+ABSL_FLAG(std::string,
+ decoder,
+ "dav1d",
+ "Decoder: dav1d, libvpx-vp9, libvpx-vp8, ffmpeg-h264, hw-vp8, "
+ "hw-vp9, hw-av1, hw-h264, hw-h265");
+ABSL_FLAG(std::string, scalability_mode, "L1T1", "Scalability mode.");
+ABSL_FLAG(int, width, 1280, "Width.");
+ABSL_FLAG(int, height, 720, "Height.");
+ABSL_FLAG(std::vector<std::string>,
+ bitrate_kbps,
+ {"1024"},
+ "Encode target bitrate per layer (l0t0,l0t1,...l1t0,l1t1 and so on) "
+ "in kbps.");
+ABSL_FLAG(double,
+ framerate_fps,
+ 30.0,
+ "Encode target frame rate of the top temporal layer in fps.");
+ABSL_FLAG(int, num_frames, 300, "Number of frames to encode and/or decode.");
+ABSL_FLAG(std::string, test_name, "", "Test name.");
ABSL_FLAG(bool, dump_decoder_input, false, "Dump decoder input.");
-
ABSL_FLAG(bool, dump_decoder_output, false, "Dump decoder output.");
-
ABSL_FLAG(bool, dump_encoder_input, false, "Dump encoder input.");
-
ABSL_FLAG(bool, dump_encoder_output, false, "Dump encoder output.");
+ABSL_FLAG(bool, write_csv, false, "Write metrics to a CSV file.");
namespace webrtc {
namespace test {
@@ -55,13 +83,58 @@
Frequency framerate;
};
-const VideoInfo kFourPeople_1280x720_30 = {
- .name = "FourPeople_1280x720_30",
- .resolution = {.width = 1280, .height = 720},
- .framerate = Frequency::Hertz(30)};
+const std::map<std::string, VideoInfo> kRawVideos = {
+ {"FourPeople_1280x720_30",
+ {.name = "FourPeople_1280x720_30",
+ .resolution = {.width = 1280, .height = 720},
+ .framerate = Frequency::Hertz(30)}},
+ {"vidyo1_1280x720_30",
+ {.name = "vidyo1_1280x720_30",
+ .resolution = {.width = 1280, .height = 720},
+ .framerate = Frequency::Hertz(30)}},
+ {"vidyo4_1280x720_30",
+ {.name = "vidyo4_1280x720_30",
+ .resolution = {.width = 1280, .height = 720},
+ .framerate = Frequency::Hertz(30)}},
+ {"KristenAndSara_1280x720_30",
+ {.name = "KristenAndSara_1280x720_30",
+ .resolution = {.width = 1280, .height = 720},
+ .framerate = Frequency::Hertz(30)}},
+ {"Johnny_1280x720_30",
+ {.name = "Johnny_1280x720_30",
+ .resolution = {.width = 1280, .height = 720},
+ .framerate = Frequency::Hertz(30)}}};
static constexpr Frequency k90kHz = Frequency::Hertz(90000);
+std::string CodecNameToCodecType(std::string name) {
+ if (name.find("av1") != std::string::npos) {
+ return "AV1";
+ }
+ if (name.find("vp9") != std::string::npos) {
+ return "VP9";
+ }
+ if (name.find("vp8") != std::string::npos) {
+ return "VP8";
+ }
+ if (name.find("h264") != std::string::npos) {
+ return "H264";
+ }
+ if (name.find("h265") != std::string::npos) {
+ return "H265";
+ }
+ RTC_CHECK_NOTREACHED();
+}
+
+// TODO(webrtc:14852): Make Create[Encoder,Decoder]Factory to work with codec
+// name directly.
+std::string CodecNameToCodecImpl(std::string name) {
+ if (name.find("hw") != std::string::npos) {
+ return "mediacodec";
+ }
+ return "builtin";
+}
+
std::unique_ptr<VideoEncoderFactory> CreateEncoderFactory(std::string impl) {
if (impl == "builtin") {
return CreateBuiltinVideoEncoderFactory();
@@ -86,10 +159,17 @@
#endif
}
+std::string TestName() {
+ std::string test_name = absl::GetFlag(FLAGS_test_name);
+ if (!test_name.empty()) {
+ return test_name;
+ }
+ return ::testing::UnitTest::GetInstance()->current_test_info()->name();
+}
+
std::string TestOutputPath() {
std::string output_path =
- OutputPath() +
- ::testing::UnitTest::GetInstance()->current_test_info()->name();
+ (rtc::StringBuilder() << OutputPath() << TestName()).str();
std::string output_dir = DirName(output_path);
bool result = CreateDir(output_dir);
RTC_CHECK(result) << "Cannot create " << output_dir;
@@ -98,7 +178,6 @@
} // namespace
std::unique_ptr<VideoCodecStats> RunEncodeDecodeTest(
- std::string codec_type,
std::string codec_impl,
const VideoInfo& video_info,
const std::map<uint32_t, EncodingSettings>& encoding_settings) {
@@ -239,7 +318,7 @@
{bitrate_kbps}, framerate_fps, num_frames);
std::unique_ptr<VideoCodecStats> stats =
- RunEncodeDecodeTest(codec_type, codec_impl, video_info, frames_settings);
+ RunEncodeDecodeTest(codec_impl, video_info, frames_settings);
VideoCodecStats::Stream stream;
if (stats != nullptr) {
@@ -252,6 +331,7 @@
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
+ /*prefix=*/"",
/*metadata=*/
{{"video_name", video_info.name},
{"codec_type", codec_type},
@@ -267,7 +347,7 @@
#else
Values("builtin"),
#endif
- Values(kFourPeople_1280x720_30),
+ Values(kRawVideos.at("FourPeople_1280x720_30")),
Values(std::make_tuple(320, 180, 30, 32, 28),
std::make_tuple(320, 180, 30, 64, 30),
std::make_tuple(320, 180, 30, 128, 33),
@@ -337,6 +417,7 @@
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
+ /*prefix=*/"",
/*metadata=*/
{{"codec_type", codec_type},
{"codec_impl", codec_impl},
@@ -345,18 +426,18 @@
std::to_string(bitrate_kbps.second)}});
}
-INSTANTIATE_TEST_SUITE_P(All,
- BitrateAdaptationTest,
- Combine(Values("AV1", "VP9", "VP8", "H264", "H265"),
+INSTANTIATE_TEST_SUITE_P(
+ All,
+ BitrateAdaptationTest,
+ Combine(Values("AV1", "VP9", "VP8", "H264", "H265"),
#if defined(WEBRTC_ANDROID)
- Values("builtin", "mediacodec"),
+ Values("builtin", "mediacodec"),
#else
- Values("builtin"),
+ Values("builtin"),
#endif
- Values(kFourPeople_1280x720_30),
- Values(std::pair(1024, 512),
- std::pair(512, 1024))),
- BitrateAdaptationTest::TestParamsToString);
+ Values(kRawVideos.at("FourPeople_1280x720_30")),
+ Values(std::pair(1024, 512), std::pair(512, 1024))),
+ BitrateAdaptationTest::TestParamsToString);
class FramerateAdaptationTest
: public ::testing::TestWithParam<std::tuple</*codec_type=*/std::string,
@@ -416,6 +497,7 @@
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
+ /*prefix=*/"",
/*metadata=*/
{{"codec_type", codec_type},
{"codec_impl", codec_impl},
@@ -424,17 +506,72 @@
std::to_string(framerate_fps.second)}});
}
-INSTANTIATE_TEST_SUITE_P(All,
- FramerateAdaptationTest,
- Combine(Values("AV1", "VP9", "VP8", "H264", "H265"),
+INSTANTIATE_TEST_SUITE_P(
+ All,
+ FramerateAdaptationTest,
+ Combine(Values("AV1", "VP9", "VP8", "H264", "H265"),
#if defined(WEBRTC_ANDROID)
- Values("builtin", "mediacodec"),
+ Values("builtin", "mediacodec"),
#else
- Values("builtin"),
+ Values("builtin"),
#endif
- Values(kFourPeople_1280x720_30),
- Values(std::pair(30, 15), std::pair(15, 30))),
- FramerateAdaptationTest::TestParamsToString);
+ Values(kRawVideos.at("FourPeople_1280x720_30")),
+ Values(std::pair(30, 15), std::pair(15, 30))),
+ FramerateAdaptationTest::TestParamsToString);
+
+TEST(VideoCodecTest, DISABLED_EncodeDecode) {
+ std::vector<std::string> bitrate_str = absl::GetFlag(FLAGS_bitrate_kbps);
+ std::vector<int> bitrate_kbps;
+ std::transform(bitrate_str.begin(), bitrate_str.end(),
+ std::back_inserter(bitrate_kbps),
+ [](const std::string& str) { return std::stoi(str); });
+
+ std::map<uint32_t, EncodingSettings> frames_settings =
+ VideoCodecTester::CreateEncodingSettings(
+ CodecNameToCodecType(absl::GetFlag(FLAGS_encoder)),
+ absl::GetFlag(FLAGS_scalability_mode), absl::GetFlag(FLAGS_width),
+ absl::GetFlag(FLAGS_height), {bitrate_kbps},
+ absl::GetFlag(FLAGS_framerate_fps), absl::GetFlag(FLAGS_num_frames));
+
+ // TODO(webrtc:14852): Pass encoder and decoder names directly, and update
+ // logged test name (implies lossing history in the chromeperf dashboard).
+ // Sync with changes in Stream::LogMetrics (see TODOs there).
+ std::unique_ptr<VideoCodecStats> stats = RunEncodeDecodeTest(
+ CodecNameToCodecImpl(absl::GetFlag(FLAGS_encoder)),
+ kRawVideos.at(absl::GetFlag(FLAGS_video_name)), frames_settings);
+ ASSERT_NE(nullptr, stats);
+
+ // Log unsliced metrics.
+ VideoCodecStats::Stream stream = stats->Aggregate(Filter{});
+ stream.LogMetrics(GetGlobalMetricsLogger(), TestName(), /*prefix=*/"",
+ /*metadata=*/{});
+
+ // Log metrics sliced on spatial and temporal layer.
+ ScalabilityMode scalability_mode =
+ *ScalabilityModeFromString(absl::GetFlag(FLAGS_scalability_mode));
+ int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode);
+ int num_temporal_layers =
+ ScalabilityModeToNumTemporalLayers(scalability_mode);
+ for (int sidx = 0; sidx < num_spatial_layers; ++sidx) {
+ for (int tidx = 0; tidx < num_temporal_layers; ++tidx) {
+ std::string metric_name_prefix =
+ (rtc::StringBuilder() << "s" << sidx << "t" << tidx << "_").str();
+ stream = stats->Aggregate(
+ {.layer_id = {{.spatial_idx = sidx, .temporal_idx = tidx}}});
+ stream.LogMetrics(GetGlobalMetricsLogger(), TestName(),
+ metric_name_prefix,
+ /*metadata=*/{});
+ }
+ }
+
+ if (absl::GetFlag(FLAGS_write_csv)) {
+ stats->LogMetrics(
+ (rtc::StringBuilder() << TestOutputPath() << ".csv").str(),
+ stats->Slice(Filter{}, /*merge=*/false), /*metadata=*/
+ {{"test_name", TestName()}});
+ }
+}
+
} // namespace test
} // namespace webrtc
diff --git a/test/BUILD.gn b/test/BUILD.gn
index fbc1ab1..e0afa27 100644
--- a/test/BUILD.gn
+++ b/test/BUILD.gn
@@ -1408,6 +1408,7 @@
"../rtc_base:checks",
"../rtc_base:logging",
"../rtc_base:rtc_event",
+ "../rtc_base:stringutils",
"../rtc_base:task_queue_for_test",
"../rtc_base:timeutils",
"../rtc_base/synchronization:mutex",
diff --git a/test/video_codec_tester.cc b/test/video_codec_tester.cc
index 26f0a61..9453c3a 100644
--- a/test/video_codec_tester.cc
+++ b/test/video_codec_tester.cc
@@ -34,6 +34,7 @@
#include "modules/video_coding/utility/ivf_file_writer.h"
#include "rtc_base/event.h"
#include "rtc_base/logging.h"
+#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/task_queue_for_test.h"
#include "rtc_base/time_utils.h"
@@ -190,8 +191,9 @@
task_queue_.PostTask([this, task = std::move(task), start]() mutable {
// `TaskQueue` doesn't guarantee FIFO order of execution for delayed
// tasks.
- int wait_ms = static_cast<int>(start.ms() - rtc::TimeMillis());
+ int64_t wait_ms = (start - Timestamp::Millis(rtc::TimeMillis())).ms();
if (wait_ms > 0) {
+ RTC_CHECK_LT(wait_ms, 10000) << "Too high wait_ms " << wait_ms;
SleepMs(wait_ms);
}
std::move(task)();
@@ -207,7 +209,7 @@
}
void PostTaskAndWait(absl::AnyInvocable<void() &&> task) {
- PostScheduledTask(std::move(task), Timestamp::Zero());
+ PostScheduledTask(std::move(task), Timestamp::Millis(rtc::TimeMillis()));
task_queue_.WaitForPreviouslyPostedTasks();
}
@@ -455,10 +457,12 @@
continue;
}
if (filter.layer_id) {
- if ((is_svc &&
- frame.layer_id.spatial_idx > filter.layer_id->spatial_idx) ||
- (!is_svc &&
- frame.layer_id.spatial_idx != filter.layer_id->spatial_idx)) {
+ if (is_svc &&
+ frame.layer_id.spatial_idx > filter.layer_id->spatial_idx) {
+ continue;
+ }
+ if (!is_svc &&
+ frame.layer_id.spatial_idx != filter.layer_id->spatial_idx) {
continue;
}
if (frame.layer_id.temporal_idx > filter.layer_id->temporal_idx) {
@@ -592,6 +596,61 @@
return stream;
}
+ void LogMetrics(absl::string_view csv_path,
+ std::vector<Frame> frames,
+ std::map<std::string, std::string> metadata) const {
+ RTC_LOG(LS_INFO) << "Write metrics to " << csv_path;
+ FILE* csv_file = fopen(csv_path.data(), "w");
+ const std::string delimiter = ";";
+ rtc::StringBuilder header;
+ header
+ << "timestamp_rtp;spatial_idx;temporal_idx;width;height;frame_size_"
+ "bytes;keyframe;qp;encode_time_us;decode_time_us;psnr_y_db;psnr_u_"
+ "db;psnr_v_db;target_bitrate_kbps;target_framerate_fps";
+ for (const auto& data : metadata) {
+ header << ";" << data.first;
+ }
+ fwrite(header.str().c_str(), 1, header.size(), csv_file);
+
+ for (const Frame& f : frames) {
+ rtc::StringBuilder row;
+ row << "\n" << f.timestamp_rtp;
+ row << ";" << f.layer_id.spatial_idx;
+ row << ";" << f.layer_id.temporal_idx;
+ row << ";" << f.width;
+ row << ";" << f.height;
+ row << ";" << f.frame_size.bytes();
+ row << ";" << f.keyframe;
+ row << ";";
+ if (f.qp) {
+ row << *f.qp;
+ }
+ row << ";" << f.encode_time.us();
+ row << ";" << f.decode_time.us();
+ if (f.psnr) {
+ row << ";" << f.psnr->y;
+ row << ";" << f.psnr->u;
+ row << ";" << f.psnr->v;
+ } else {
+ row << ";;;";
+ }
+
+ const auto& es = encoding_settings_.at(f.timestamp_rtp);
+ row << ";"
+ << f.target_bitrate.value_or(GetTargetBitrate(es, f.layer_id)).kbps();
+ row << ";"
+ << f.target_framerate.value_or(GetTargetFramerate(es, f.layer_id))
+ .hertz<double>();
+
+ for (const auto& data : metadata) {
+ row << ";" << data.second;
+ }
+ fwrite(row.str().c_str(), 1, row.size(), csv_file);
+ }
+
+ fclose(csv_file);
+ }
+
void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); }
private:
@@ -1077,55 +1136,60 @@
void VideoCodecStats::Stream::LogMetrics(
MetricsLogger* logger,
std::string test_case_name,
+ std::string prefix,
std::map<std::string, std::string> metadata) const {
- logger->LogMetric("width", test_case_name, width, Unit::kCount,
+ logger->LogMetric(prefix + "width", test_case_name, width, Unit::kCount,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric("height", test_case_name, height, Unit::kCount,
+ logger->LogMetric(prefix + "height", test_case_name, height, Unit::kCount,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric("frame_size_bytes", test_case_name, frame_size_bytes,
- Unit::kBytes, ImprovementDirection::kNeitherIsBetter,
- metadata);
- logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount,
+ logger->LogMetric(prefix + "frame_size_bytes", test_case_name,
+ frame_size_bytes, Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter, metadata);
+ logger->LogMetric(prefix + "keyframe", test_case_name, keyframe, Unit::kCount,
ImprovementDirection::kSmallerIsBetter, metadata);
- logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless,
+ logger->LogMetric(prefix + "qp", test_case_name, qp, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter, metadata);
- logger->LogMetric("encode_time_ms", test_case_name, encode_time_ms,
+ // TODO(webrtc:14852): Change to us or even ns.
+ logger->LogMetric(prefix + "encode_time_ms", test_case_name, encode_time_ms,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter,
metadata);
- logger->LogMetric("decode_time_ms", test_case_name, decode_time_ms,
+ logger->LogMetric(prefix + "decode_time_ms", test_case_name, decode_time_ms,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter,
metadata);
// TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted
// to bytes per second in Chromeperf dash.
- logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps,
- Unit::kKilobitsPerSecond,
+ logger->LogMetric(prefix + "target_bitrate_kbps", test_case_name,
+ target_bitrate_kbps, Unit::kKilobitsPerSecond,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric("target_framerate_fps", test_case_name,
+ logger->LogMetric(prefix + "target_framerate_fps", test_case_name,
target_framerate_fps, Unit::kHertz,
ImprovementDirection::kBiggerIsBetter, metadata);
// TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted
// to bytes per second in Chromeperf dash.
- logger->LogMetric("encoded_bitrate_kbps", test_case_name,
+ logger->LogMetric(prefix + "encoded_bitrate_kbps", test_case_name,
encoded_bitrate_kbps, Unit::kKilobitsPerSecond,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric("encoded_framerate_fps", test_case_name,
+ logger->LogMetric(prefix + "encoded_framerate_fps", test_case_name,
encoded_framerate_fps, Unit::kHertz,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric("bitrate_mismatch_pct", test_case_name,
+ logger->LogMetric(prefix + "bitrate_mismatch_pct", test_case_name,
bitrate_mismatch_pct, Unit::kPercent,
ImprovementDirection::kNeitherIsBetter, metadata);
- logger->LogMetric("framerate_mismatch_pct", test_case_name,
+ logger->LogMetric(prefix + "framerate_mismatch_pct", test_case_name,
framerate_mismatch_pct, Unit::kPercent,
ImprovementDirection::kNeitherIsBetter, metadata);
- logger->LogMetric("transmission_time_ms", test_case_name,
+ logger->LogMetric(prefix + "transmission_time_ms", test_case_name,
transmission_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter, metadata);
- logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless,
- ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless,
- ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless,
- ImprovementDirection::kBiggerIsBetter, metadata);
+ logger->LogMetric(prefix + "psnr_y_db", test_case_name, psnr.y,
+ Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
+ metadata);
+ logger->LogMetric(prefix + "psnr_u_db", test_case_name, psnr.u,
+ Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
+ metadata);
+ logger->LogMetric(prefix + "psnr_v_db", test_case_name, psnr.v,
+ Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
+ metadata);
}
// TODO(ssilkin): use Frequency and DataRate for framerate and bitrate.
diff --git a/test/video_codec_tester.h b/test/video_codec_tester.h
index dc72645..87cc5f7 100644
--- a/test/video_codec_tester.h
+++ b/test/video_codec_tester.h
@@ -68,7 +68,6 @@
};
struct Frame {
- int frame_num = 0;
uint32_t timestamp_rtp = 0;
LayerId layer_id;
bool encoded = false;
@@ -118,6 +117,7 @@
// Logs `Stream` metrics to provided `MetricsLogger`.
void LogMetrics(MetricsLogger* logger,
std::string test_case_name,
+ std::string prefix,
std::map<std::string, std::string> metadata = {}) const;
};
@@ -130,6 +130,12 @@
// Returns video statistics aggregated for the slice specified by `filter`.
virtual Stream Aggregate(Filter filter) const = 0;
+
+ // Write metrics to a CSV file.
+ virtual void LogMetrics(
+ absl::string_view csv_path,
+ std::vector<Frame> frames,
+ std::map<std::string, std::string> metadata) const = 0;
};
// Pacing settings for codec input.
