Revert "Added an encode/decode test parameterizable via command line"
This reverts commit 496893e89e5bc8139e50befcb1a26eadbd829b0d.
Reason for revert: Breaks https://ci.chromium.org/ui/p/chromium/builders/webrtc.fyi/WebRTC%20Chromium%20FYI%20ios-device/16103/overview
Original change's description:
> Added an encode/decode test parameterizable via command line
>
> This enables testing different settings without updating code and rebuilding the test binary. Example of command:
>
> video_codec_perf_tests --gtest_also_run_disabled_tests --gtest_filter=*EncodeDecode --encoder=libaom-av1 --decoder=dav1d --scalability_mode=L1T3 --bitrate_kbps=100,200,300 --framerate_fps=30 --write_csv
>
> Also added writing per-frame stats to a CSV. It is more convenient to work with CSV than to parse metrics proto.
>
> Bug: webrtc:14852
> Change-Id: I1b3970f7ffa88c016133197aff585de5bc4e35c6
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/327600
> Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#41179}
Bug: webrtc:14852
Change-Id: Ifdce738058c3e3fc7c76886939add2813405cae7
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/327722
Owners-Override: Christoffer Jansson <jansson@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Christoffer Jansson <jansson@webrtc.org>
Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
Cr-Commit-Position: refs/heads/main@{#41183}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index d9e614f..c75b433 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -1005,9 +1005,7 @@
"../../api/video:resolution",
"../../api/video_codecs:builtin_video_decoder_factory",
"../../api/video_codecs:builtin_video_encoder_factory",
- "../../modules/video_coding/svc:scalability_mode_util",
"../../rtc_base:logging",
- "../../rtc_base:stringutils",
"../../test:fileutils",
"../../test:test_flags",
"../../test:test_main",
diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc
index 055d853..08961a3 100644
--- a/modules/video_coding/codecs/test/video_codec_test.cc
+++ b/modules/video_coding/codecs/test/video_codec_test.cc
@@ -23,48 +23,19 @@
#if defined(WEBRTC_ANDROID)
#include "modules/video_coding/codecs/test/android_codec_factory_helper.h"
#endif
-#include "modules/video_coding/svc/scalability_mode_util.h"
#include "rtc_base/logging.h"
-#include "rtc_base/strings/str_join.h"
-#include "rtc_base/strings/string_builder.h"
#include "test/gtest.h"
#include "test/test_flags.h"
#include "test/testsupport/file_utils.h"
#include "test/video_codec_tester.h"
-ABSL_FLAG(std::string,
- video_name,
- "FourPeople_1280x720_30",
- "Name of input video sequence.");
-ABSL_FLAG(std::string,
- encoder,
- "libaom-av1",
- "Encoder: libaom-av1, libvpx-vp9, libvpx-vp8, openh264, hw-vp8, "
- "hw-vp9, hw-av1, hw-h264, hw-h265");
-ABSL_FLAG(std::string,
- decoder,
- "dav1d",
- "Decoder: dav1d, libvpx-vp9, libvpx-vp8, ffmpeg-h264, hw-vp8, "
- "hw-vp9, hw-av1, hw-h264, hw-h265");
-ABSL_FLAG(std::string, scalability_mode, "L1T1", "Scalability mode.");
-ABSL_FLAG(int, width, 1280, "Width.");
-ABSL_FLAG(int, height, 720, "Height.");
-ABSL_FLAG(std::vector<std::string>,
- bitrate_kbps,
- {"1024"},
- "Encode target bitrate per layer (l0t0,l0t1,...l1t0,l1t1 and so on) "
- "in kbps.");
-ABSL_FLAG(double,
- framerate_fps,
- 30.0,
- "Encode target frame rate of the top temporal layer in fps.");
-ABSL_FLAG(int, num_frames, 300, "Number of frames to encode and/or decode.");
-ABSL_FLAG(std::string, test_name, "", "Test name.");
ABSL_FLAG(bool, dump_decoder_input, false, "Dump decoder input.");
+
ABSL_FLAG(bool, dump_decoder_output, false, "Dump decoder output.");
+
ABSL_FLAG(bool, dump_encoder_input, false, "Dump encoder input.");
+
ABSL_FLAG(bool, dump_encoder_output, false, "Dump encoder output.");
-ABSL_FLAG(bool, write_csv, false, "Write metrics to a CSV file.");
namespace webrtc {
namespace test {
@@ -84,58 +55,13 @@
Frequency framerate;
};
-const std::map<std::string, VideoInfo> kRawVideos = {
- {"FourPeople_1280x720_30",
- {.name = "FourPeople_1280x720_30",
- .resolution = {.width = 1280, .height = 720},
- .framerate = Frequency::Hertz(30)}},
- {"vidyo1_1280x720_30",
- {.name = "vidyo1_1280x720_30",
- .resolution = {.width = 1280, .height = 720},
- .framerate = Frequency::Hertz(30)}},
- {"vidyo4_1280x720_30",
- {.name = "vidyo4_1280x720_30",
- .resolution = {.width = 1280, .height = 720},
- .framerate = Frequency::Hertz(30)}},
- {"KristenAndSara_1280x720_30",
- {.name = "KristenAndSara_1280x720_30",
- .resolution = {.width = 1280, .height = 720},
- .framerate = Frequency::Hertz(30)}},
- {"Johnny_1280x720_30",
- {.name = "Johnny_1280x720_30",
- .resolution = {.width = 1280, .height = 720},
- .framerate = Frequency::Hertz(30)}}};
+const VideoInfo kFourPeople_1280x720_30 = {
+ .name = "FourPeople_1280x720_30",
+ .resolution = {.width = 1280, .height = 720},
+ .framerate = Frequency::Hertz(30)};
static constexpr Frequency k90kHz = Frequency::Hertz(90000);
-std::string CodecNameToCodecType(std::string name) {
- if (name.find("av1") != std::string::npos) {
- return "AV1";
- }
- if (name.find("vp9") != std::string::npos) {
- return "VP9";
- }
- if (name.find("vp8") != std::string::npos) {
- return "VP8";
- }
- if (name.find("h264") != std::string::npos) {
- return "H264";
- }
- if (name.find("h265") != std::string::npos) {
- return "H265";
- }
- RTC_CHECK_NOTREACHED();
-}
-
-// TODO(webrtc:14852): Make Create[Encoder,Decoder]Factory to work with codec
-// name directly.
-std::string CodecNameToCodecImpl(std::string name) {
- if (name.find("hw") != std::string::npos) {
- return "mediacodec";
- }
- return "builtin";
-}
-
std::unique_ptr<VideoEncoderFactory> CreateEncoderFactory(std::string impl) {
if (impl == "builtin") {
return CreateBuiltinVideoEncoderFactory();
@@ -160,17 +86,10 @@
#endif
}
-std::string TestName() {
- std::string test_name = absl::GetFlag(FLAGS_test_name);
- if (!test_name.empty()) {
- return test_name;
- }
- return ::testing::UnitTest::GetInstance()->current_test_info()->name();
-}
-
std::string TestOutputPath() {
std::string output_path =
- (rtc::StringBuilder() << OutputPath() << TestName()).str();
+ OutputPath() +
+ ::testing::UnitTest::GetInstance()->current_test_info()->name();
std::string output_dir = DirName(output_path);
bool result = CreateDir(output_dir);
RTC_CHECK(result) << "Cannot create " << output_dir;
@@ -179,6 +98,7 @@
} // namespace
std::unique_ptr<VideoCodecStats> RunEncodeDecodeTest(
+ std::string codec_type,
std::string codec_impl,
const VideoInfo& video_info,
const std::map<uint32_t, EncodingSettings>& encoding_settings) {
@@ -319,7 +239,7 @@
{bitrate_kbps}, framerate_fps, num_frames);
std::unique_ptr<VideoCodecStats> stats =
- RunEncodeDecodeTest(codec_impl, video_info, frames_settings);
+ RunEncodeDecodeTest(codec_type, codec_impl, video_info, frames_settings);
VideoCodecStats::Stream stream;
if (stats != nullptr) {
@@ -332,7 +252,6 @@
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
- /*prefix=*/"",
/*metadata=*/
{{"video_name", video_info.name},
{"codec_type", codec_type},
@@ -348,7 +267,7 @@
#else
Values("builtin"),
#endif
- Values(kRawVideos.at("FourPeople_1280x720_30")),
+ Values(kFourPeople_1280x720_30),
Values(std::make_tuple(320, 180, 30, 32, 28),
std::make_tuple(320, 180, 30, 64, 30),
std::make_tuple(320, 180, 30, 128, 33),
@@ -418,7 +337,6 @@
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
- /*prefix=*/"",
/*metadata=*/
{{"codec_type", codec_type},
{"codec_impl", codec_impl},
@@ -427,18 +345,18 @@
std::to_string(bitrate_kbps.second)}});
}
-INSTANTIATE_TEST_SUITE_P(
- All,
- BitrateAdaptationTest,
- Combine(Values("AV1", "VP9", "VP8", "H264", "H265"),
+INSTANTIATE_TEST_SUITE_P(All,
+ BitrateAdaptationTest,
+ Combine(Values("AV1", "VP9", "VP8", "H264", "H265"),
#if defined(WEBRTC_ANDROID)
- Values("builtin", "mediacodec"),
+ Values("builtin", "mediacodec"),
#else
- Values("builtin"),
+ Values("builtin"),
#endif
- Values(kRawVideos.at("FourPeople_1280x720_30")),
- Values(std::pair(1024, 512), std::pair(512, 1024))),
- BitrateAdaptationTest::TestParamsToString);
+ Values(kFourPeople_1280x720_30),
+ Values(std::pair(1024, 512),
+ std::pair(512, 1024))),
+ BitrateAdaptationTest::TestParamsToString);
class FramerateAdaptationTest
: public ::testing::TestWithParam<std::tuple</*codec_type=*/std::string,
@@ -498,7 +416,6 @@
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
- /*prefix=*/"",
/*metadata=*/
{{"codec_type", codec_type},
{"codec_impl", codec_impl},
@@ -507,72 +424,17 @@
std::to_string(framerate_fps.second)}});
}
-INSTANTIATE_TEST_SUITE_P(
- All,
- FramerateAdaptationTest,
- Combine(Values("AV1", "VP9", "VP8", "H264", "H265"),
+INSTANTIATE_TEST_SUITE_P(All,
+ FramerateAdaptationTest,
+ Combine(Values("AV1", "VP9", "VP8", "H264", "H265"),
#if defined(WEBRTC_ANDROID)
- Values("builtin", "mediacodec"),
+ Values("builtin", "mediacodec"),
#else
- Values("builtin"),
+ Values("builtin"),
#endif
- Values(kRawVideos.at("FourPeople_1280x720_30")),
- Values(std::pair(30, 15), std::pair(15, 30))),
- FramerateAdaptationTest::TestParamsToString);
-
-TEST(VideoCodecTest, DISABLED_EncodeDecode) {
- std::vector<std::string> bitrate_str = absl::GetFlag(FLAGS_bitrate_kbps);
- std::vector<int> bitrate_kbps;
- std::transform(bitrate_str.begin(), bitrate_str.end(),
- std::back_inserter(bitrate_kbps),
- [](const std::string& str) { return std::stoi(str); });
-
- std::map<uint32_t, EncodingSettings> frames_settings =
- VideoCodecTester::CreateEncodingSettings(
- CodecNameToCodecType(absl::GetFlag(FLAGS_encoder)),
- absl::GetFlag(FLAGS_scalability_mode), absl::GetFlag(FLAGS_width),
- absl::GetFlag(FLAGS_height), {bitrate_kbps},
- absl::GetFlag(FLAGS_framerate_fps), absl::GetFlag(FLAGS_num_frames));
-
- // TODO(webrtc:14852): Pass encoder and decoder names directly, and update
- // logged test name (implies lossing history in the chromeperf dashboard).
- // Sync with changes in Stream::LogMetrics (see TODOs there).
- std::unique_ptr<VideoCodecStats> stats = RunEncodeDecodeTest(
- CodecNameToCodecImpl(absl::GetFlag(FLAGS_encoder)),
- kRawVideos.at(absl::GetFlag(FLAGS_video_name)), frames_settings);
- ASSERT_NE(nullptr, stats);
-
- // Log unsliced metrics.
- VideoCodecStats::Stream stream = stats->Aggregate(Filter{});
- stream.LogMetrics(GetGlobalMetricsLogger(), TestName(), /*prefix=*/"",
- /*metadata=*/{});
-
- // Log metrics sliced on spatial and temporal layer.
- ScalabilityMode scalability_mode =
- *ScalabilityModeFromString(absl::GetFlag(FLAGS_scalability_mode));
- int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode);
- int num_temporal_layers =
- ScalabilityModeToNumTemporalLayers(scalability_mode);
- for (int sidx = 0; sidx < num_spatial_layers; ++sidx) {
- for (int tidx = 0; tidx < num_temporal_layers; ++tidx) {
- std::string metric_name_prefix =
- (rtc::StringBuilder() << "s" << sidx << "t" << tidx << "_").str();
- stream = stats->Aggregate(
- {.layer_id = {{.spatial_idx = sidx, .temporal_idx = tidx}}});
- stream.LogMetrics(GetGlobalMetricsLogger(), TestName(),
- metric_name_prefix,
- /*metadata=*/{});
- }
- }
-
- if (absl::GetFlag(FLAGS_write_csv)) {
- stats->LogMetrics(
- (rtc::StringBuilder() << TestOutputPath() << ".csv").str(),
- stats->Slice(Filter{}, /*merge=*/false), /*metadata=*/
- {{"test_name", TestName()}});
- }
-}
-
+ Values(kFourPeople_1280x720_30),
+ Values(std::pair(30, 15), std::pair(15, 30))),
+ FramerateAdaptationTest::TestParamsToString);
} // namespace test
} // namespace webrtc
diff --git a/test/video_codec_tester.cc b/test/video_codec_tester.cc
index 38d97ea..26f0a61 100644
--- a/test/video_codec_tester.cc
+++ b/test/video_codec_tester.cc
@@ -455,12 +455,10 @@
continue;
}
if (filter.layer_id) {
- if (is_svc &&
- frame.layer_id.spatial_idx > filter.layer_id->spatial_idx) {
- continue;
- }
- if (!is_svc &&
- frame.layer_id.spatial_idx != filter.layer_id->spatial_idx) {
+ if ((is_svc &&
+ frame.layer_id.spatial_idx > filter.layer_id->spatial_idx) ||
+ (!is_svc &&
+ frame.layer_id.spatial_idx != filter.layer_id->spatial_idx)) {
continue;
}
if (frame.layer_id.temporal_idx > filter.layer_id->temporal_idx) {
@@ -594,61 +592,6 @@
return stream;
}
- void LogMetrics(absl::string_view csv_path,
- std::vector<Frame> frames,
- std::map<std::string, std::string> metadata) const {
- RTC_LOG(LS_INFO) << "Write metrics to " << csv_path;
- FILE* csv_file = fopen(csv_path.data(), "w");
- const std::string delimiter = ";";
- rtc::StringBuilder header;
- header
- << "timestamp_rtp;spatial_idx;temporal_idx;width;height;frame_size_"
- "bytes;keyframe;qp;encode_time_us;decode_time_us;psnr_y_db;psnr_u_"
- "db;psnr_v_db;target_bitrate_kbps;target_framerate_fps";
- for (const auto& data : metadata) {
- header << ";" << data.first;
- }
- fwrite(header.str().c_str(), 1, header.size(), csv_file);
-
- for (const Frame& f : frames) {
- rtc::StringBuilder row;
- row << "\n" << f.timestamp_rtp;
- row << ";" << f.layer_id.spatial_idx;
- row << ";" << f.layer_id.temporal_idx;
- row << ";" << f.width;
- row << ";" << f.height;
- row << ";" << f.frame_size.bytes();
- row << ";" << f.keyframe;
- row << ";";
- if (f.qp) {
- row << *f.qp;
- }
- row << ";" << f.encode_time.us();
- row << ";" << f.decode_time.us();
- if (f.psnr) {
- row << ";" << f.psnr->y;
- row << ";" << f.psnr->u;
- row << ";" << f.psnr->v;
- } else {
- row << ";;;";
- }
-
- const auto& es = encoding_settings_.at(f.timestamp_rtp);
- row << ";"
- << f.target_bitrate.value_or(GetTargetBitrate(es, f.layer_id)).kbps();
- row << ";"
- << f.target_framerate.value_or(GetTargetFramerate(es, f.layer_id))
- .hertz<double>();
-
- for (const auto& data : metadata) {
- row << ";" << data.second;
- }
- fwrite(row.str().c_str(), 1, row.size(), csv_file);
- }
-
- fclose(csv_file);
- }
-
void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); }
private:
@@ -1134,60 +1077,55 @@
void VideoCodecStats::Stream::LogMetrics(
MetricsLogger* logger,
std::string test_case_name,
- std::string prefix,
std::map<std::string, std::string> metadata) const {
- logger->LogMetric(prefix + "width", test_case_name, width, Unit::kCount,
+ logger->LogMetric("width", test_case_name, width, Unit::kCount,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric(prefix + "height", test_case_name, height, Unit::kCount,
+ logger->LogMetric("height", test_case_name, height, Unit::kCount,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric(prefix + "frame_size_bytes", test_case_name,
- frame_size_bytes, Unit::kBytes,
- ImprovementDirection::kNeitherIsBetter, metadata);
- logger->LogMetric(prefix + "keyframe", test_case_name, keyframe, Unit::kCount,
+ logger->LogMetric("frame_size_bytes", test_case_name, frame_size_bytes,
+ Unit::kBytes, ImprovementDirection::kNeitherIsBetter,
+ metadata);
+ logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount,
ImprovementDirection::kSmallerIsBetter, metadata);
- logger->LogMetric(prefix + "qp", test_case_name, qp, Unit::kUnitless,
+ logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter, metadata);
- // TODO(webrtc:14852): Change to us or even ns.
- logger->LogMetric(prefix + "encode_time_ms", test_case_name, encode_time_ms,
+ logger->LogMetric("encode_time_ms", test_case_name, encode_time_ms,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter,
metadata);
- logger->LogMetric(prefix + "decode_time_ms", test_case_name, decode_time_ms,
+ logger->LogMetric("decode_time_ms", test_case_name, decode_time_ms,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter,
metadata);
// TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted
// to bytes per second in Chromeperf dash.
- logger->LogMetric(prefix + "target_bitrate_kbps", test_case_name,
- target_bitrate_kbps, Unit::kKilobitsPerSecond,
+ logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps,
+ Unit::kKilobitsPerSecond,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric(prefix + "target_framerate_fps", test_case_name,
+ logger->LogMetric("target_framerate_fps", test_case_name,
target_framerate_fps, Unit::kHertz,
ImprovementDirection::kBiggerIsBetter, metadata);
// TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted
// to bytes per second in Chromeperf dash.
- logger->LogMetric(prefix + "encoded_bitrate_kbps", test_case_name,
+ logger->LogMetric("encoded_bitrate_kbps", test_case_name,
encoded_bitrate_kbps, Unit::kKilobitsPerSecond,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric(prefix + "encoded_framerate_fps", test_case_name,
+ logger->LogMetric("encoded_framerate_fps", test_case_name,
encoded_framerate_fps, Unit::kHertz,
ImprovementDirection::kBiggerIsBetter, metadata);
- logger->LogMetric(prefix + "bitrate_mismatch_pct", test_case_name,
+ logger->LogMetric("bitrate_mismatch_pct", test_case_name,
bitrate_mismatch_pct, Unit::kPercent,
ImprovementDirection::kNeitherIsBetter, metadata);
- logger->LogMetric(prefix + "framerate_mismatch_pct", test_case_name,
+ logger->LogMetric("framerate_mismatch_pct", test_case_name,
framerate_mismatch_pct, Unit::kPercent,
ImprovementDirection::kNeitherIsBetter, metadata);
- logger->LogMetric(prefix + "transmission_time_ms", test_case_name,
+ logger->LogMetric("transmission_time_ms", test_case_name,
transmission_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter, metadata);
- logger->LogMetric(prefix + "psnr_y_db", test_case_name, psnr.y,
- Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
- metadata);
- logger->LogMetric(prefix + "psnr_u_db", test_case_name, psnr.u,
- Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
- metadata);
- logger->LogMetric(prefix + "psnr_v_db", test_case_name, psnr.v,
- Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
- metadata);
+ logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter, metadata);
+ logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter, metadata);
+ logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter, metadata);
}
// TODO(ssilkin): use Frequency and DataRate for framerate and bitrate.
diff --git a/test/video_codec_tester.h b/test/video_codec_tester.h
index 87cc5f7..dc72645 100644
--- a/test/video_codec_tester.h
+++ b/test/video_codec_tester.h
@@ -68,6 +68,7 @@
};
struct Frame {
+ int frame_num = 0;
uint32_t timestamp_rtp = 0;
LayerId layer_id;
bool encoded = false;
@@ -117,7 +118,6 @@
// Logs `Stream` metrics to provided `MetricsLogger`.
void LogMetrics(MetricsLogger* logger,
std::string test_case_name,
- std::string prefix,
std::map<std::string, std::string> metadata = {}) const;
};
@@ -130,12 +130,6 @@
// Returns video statistics aggregated for the slice specified by `filter`.
virtual Stream Aggregate(Filter filter) const = 0;
-
- // Write metrics to a CSV file.
- virtual void LogMetrics(
- absl::string_view csv_path,
- std::vector<Frame> frames,
- std::map<std::string, std::string> metadata) const = 0;
};
// Pacing settings for codec input.
