Refactoring scenario stats analysis.
This CL just moves code around to prepare for an upcoming
CL where more stats collection is added to scenario tests.
Bug: webrtc:10365
Change-Id: I8a960e44fd11fc36047677c4d8dfc0af96aacb22
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132002
Commit-Queue: Sebastian Jansson <srte@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27519}diff --git a/test/scenario/BUILD.gn b/test/scenario/BUILD.gn
index b9d0a8a..aafafa5 100644
--- a/test/scenario/BUILD.gn
+++ b/test/scenario/BUILD.gn
@@ -59,15 +59,17 @@
"hardware_codecs.h",
"network_node.cc",
"network_node.h",
- "quality_info.h",
- "quality_stats.cc",
- "quality_stats.h",
+ "performance_stats.h",
"scenario.cc",
"scenario.h",
"scenario_config.cc",
"scenario_config.h",
"simulated_time.cc",
"simulated_time.h",
+ "stats_collection.cc",
+ "stats_collection.h",
+ "video_frame_matcher.cc",
+ "video_frame_matcher.h",
"video_stream.cc",
"video_stream.h",
]
@@ -156,8 +158,8 @@
rtc_source_set("scenario_unittests") {
testonly = true
sources = [
- "quality_stats_unittest.cc",
"scenario_unittest.cc",
+ "stats_collection_unittest.cc",
"video_stream_unittest.cc",
]
deps = [
diff --git a/test/scenario/quality_info.h b/test/scenario/performance_stats.h
similarity index 64%
rename from test/scenario/quality_info.h
rename to test/scenario/performance_stats.h
index 0cf6eaf..e58dab3 100644
--- a/test/scenario/quality_info.h
+++ b/test/scenario/performance_stats.h
@@ -1,5 +1,5 @@
/*
- * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ * Copyright 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@@ -7,17 +7,21 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef TEST_SCENARIO_QUALITY_INFO_H_
-#define TEST_SCENARIO_QUALITY_INFO_H_
+#ifndef TEST_SCENARIO_PERFORMANCE_STATS_H_
+#define TEST_SCENARIO_PERFORMANCE_STATS_H_
+#include "api/units/data_rate.h"
+#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "api/video/video_frame_buffer.h"
+#include "test/statistics.h"
namespace webrtc {
namespace test {
+
struct VideoFramePair {
- rtc::scoped_refptr<webrtc::VideoFrameBuffer> captured;
- rtc::scoped_refptr<webrtc::VideoFrameBuffer> decoded;
+ rtc::scoped_refptr<VideoFrameBuffer> captured;
+ rtc::scoped_refptr<VideoFrameBuffer> decoded;
Timestamp capture_time = Timestamp::MinusInfinity();
Timestamp render_time = Timestamp::PlusInfinity();
// A unique identifier for the spatial/temporal layer the decoded frame
@@ -30,6 +34,16 @@
// decoded frame has matched differend captured frames.
int repeated = 0;
};
+
+struct VideoQualityStats {
+ int captures_count = 0;
+ int valid_count = 0;
+ int lost_count = 0;
+ Statistics end_to_end_seconds;
+ Statistics frame_size;
+ Statistics psnr;
+};
+
} // namespace test
} // namespace webrtc
-#endif // TEST_SCENARIO_QUALITY_INFO_H_
+#endif // TEST_SCENARIO_PERFORMANCE_STATS_H_
diff --git a/test/scenario/scenario_config.h b/test/scenario/scenario_config.h
index bc0fd01..84caa9b 100644
--- a/test/scenario/scenario_config.h
+++ b/test/scenario/scenario_config.h
@@ -22,7 +22,7 @@
#include "api/units/time_delta.h"
#include "api/video/video_codec_type.h"
#include "test/frame_generator.h"
-#include "test/scenario/quality_info.h"
+#include "test/scenario/performance_stats.h"
namespace webrtc {
namespace test {
diff --git a/test/scenario/scenario_unittest.cc b/test/scenario/scenario_unittest.cc
index 2b1cec6..e3abd48 100644
--- a/test/scenario/scenario_unittest.cc
+++ b/test/scenario/scenario_unittest.cc
@@ -9,8 +9,10 @@
*/
#include <atomic>
-#include "test/scenario/scenario.h"
#include "test/gtest.h"
+#include "test/scenario/scenario.h"
+#include "test/scenario/stats_collection.h"
+
namespace webrtc {
namespace test {
TEST(ScenarioTest, StartsAndStopsWithoutErrors) {
diff --git a/test/scenario/stats_collection.cc b/test/scenario/stats_collection.cc
new file mode 100644
index 0000000..77b2ae0
--- /dev/null
+++ b/test/scenario/stats_collection.cc
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/scenario/stats_collection.h"
+
+#include <utility>
+
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+
+namespace webrtc {
+namespace test {
+
+VideoQualityAnalyzer::VideoQualityAnalyzer(
+ VideoQualityAnalyzerConfig config,
+ std::unique_ptr<RtcEventLogOutput> writer)
+ : config_(config), writer_(std::move(writer)) {
+ if (writer_) {
+ PrintHeaders();
+ }
+}
+
+VideoQualityAnalyzer::~VideoQualityAnalyzer() = default;
+
+void VideoQualityAnalyzer::PrintHeaders() {
+ writer_->Write(
+ "capture_time render_time capture_width capture_height render_width "
+ "render_height psnr\n");
+}
+
+std::function<void(const VideoFramePair&)> VideoQualityAnalyzer::Handler() {
+ return [this](VideoFramePair pair) { HandleFramePair(pair); };
+}
+
+void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) {
+ double psnr = NAN;
+ RTC_CHECK(sample.captured);
+ ++stats_.captures_count;
+ if (!sample.decoded) {
+ ++stats_.lost_count;
+ } else {
+ psnr = I420PSNR(*sample.captured->ToI420(), *sample.decoded->ToI420());
+ ++stats_.valid_count;
+ stats_.end_to_end_seconds.AddSample(
+ (sample.render_time - sample.capture_time).seconds<double>());
+ stats_.psnr.AddSample(psnr);
+ }
+ if (writer_) {
+ LogWriteFormat(writer_.get(), "%.3f %.3f %.3f %i %i %i %i %.3f\n",
+ sample.capture_time.seconds<double>(),
+ sample.render_time.seconds<double>(),
+ sample.captured->width(), sample.captured->height(),
+ sample.decoded->width(), sample.decoded->height(), psnr);
+ }
+}
+
+VideoQualityStats VideoQualityAnalyzer::stats() const {
+ return stats_;
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/test/scenario/stats_collection.h b/test/scenario/stats_collection.h
new file mode 100644
index 0000000..d1b46e4
--- /dev/null
+++ b/test/scenario/stats_collection.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef TEST_SCENARIO_STATS_COLLECTION_H_
+#define TEST_SCENARIO_STATS_COLLECTION_H_
+
+#include <memory>
+
+#include "test/logging/log_writer.h"
+#include "test/scenario/performance_stats.h"
+
+namespace webrtc {
+namespace test {
+
+struct VideoQualityAnalyzerConfig {
+ double psnr_coverage = 1;
+};
+
+class VideoQualityAnalyzer {
+ public:
+ explicit VideoQualityAnalyzer(
+ VideoQualityAnalyzerConfig config = VideoQualityAnalyzerConfig(),
+ std::unique_ptr<RtcEventLogOutput> writer = nullptr);
+ ~VideoQualityAnalyzer();
+ void HandleFramePair(VideoFramePair sample);
+ VideoQualityStats stats() const;
+ void PrintHeaders();
+ void PrintFrameInfo(const VideoFramePair& sample);
+ std::function<void(const VideoFramePair&)> Handler();
+
+ private:
+ const VideoQualityAnalyzerConfig config_;
+ VideoQualityStats stats_;
+ const std::unique_ptr<RtcEventLogOutput> writer_;
+};
+
+} // namespace test
+} // namespace webrtc
+
+#endif // TEST_SCENARIO_STATS_COLLECTION_H_
diff --git a/test/scenario/quality_stats_unittest.cc b/test/scenario/stats_collection_unittest.cc
similarity index 97%
rename from test/scenario/quality_stats_unittest.cc
rename to test/scenario/stats_collection_unittest.cc
index bef9f4b..ee01ad5 100644
--- a/test/scenario/quality_stats_unittest.cc
+++ b/test/scenario/stats_collection_unittest.cc
@@ -7,6 +7,7 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
+#include "test/scenario/stats_collection.h"
#include "test/gtest.h"
#include "test/scenario/scenario.h"
diff --git a/test/scenario/quality_stats.cc b/test/scenario/video_frame_matcher.cc
similarity index 73%
rename from test/scenario/quality_stats.cc
rename to test/scenario/video_frame_matcher.cc
index 61d6edd..c0d7925 100644
--- a/test/scenario/quality_stats.cc
+++ b/test/scenario/video_frame_matcher.cc
@@ -7,7 +7,7 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "test/scenario/quality_stats.h"
+#include "test/scenario/video_frame_matcher.h"
#include <utility>
@@ -83,7 +83,7 @@
}
while (!layer.captured_frames.empty() &&
layer.captured_frames.front().matched) {
- HandleMatch(layer.captured_frames.front(), layer_id);
+ HandleMatch(std::move(layer.captured_frames.front()), layer_id);
layer.captured_frames.pop_front();
}
});
@@ -93,10 +93,27 @@
return !frame_pair_handlers_.empty();
}
+void VideoFrameMatcher::HandleMatch(VideoFrameMatcher::CapturedFrame captured,
+ int layer_id) {
+ VideoFramePair frame_pair;
+ frame_pair.layer_id = layer_id;
+ frame_pair.captured = captured.frame;
+ frame_pair.capture_id = captured.id;
+ if (captured.best_decode) {
+ frame_pair.decode_id = captured.best_decode->id;
+ frame_pair.capture_time = captured.capture_time;
+ frame_pair.decoded = captured.best_decode->frame;
+ frame_pair.render_time = captured.best_decode->render_time;
+ frame_pair.repeated = captured.best_decode->repeat_count++;
+ }
+ for (auto& handler : frame_pair_handlers_)
+ handler(frame_pair);
+}
+
void VideoFrameMatcher::Finalize() {
for (auto& layer : layers_) {
while (!layer.second.captured_frames.empty()) {
- HandleMatch(layer.second.captured_frames.front(), layer.first);
+ HandleMatch(std::move(layer.second.captured_frames.front()), layer.first);
layer.second.captured_frames.pop_front();
}
}
@@ -143,52 +160,5 @@
layer_id_);
}
-VideoQualityAnalyzer::VideoQualityAnalyzer(
- VideoQualityAnalyzerConfig config,
- std::unique_ptr<RtcEventLogOutput> writer)
- : config_(config), writer_(std::move(writer)) {
- if (writer_) {
- PrintHeaders();
- }
-}
-
-VideoQualityAnalyzer::~VideoQualityAnalyzer() = default;
-
-void VideoQualityAnalyzer::PrintHeaders() {
- writer_->Write(
- "capture_time render_time capture_width capture_height render_width "
- "render_height psnr\n");
-}
-
-std::function<void(const VideoFramePair&)> VideoQualityAnalyzer::Handler() {
- return [this](VideoFramePair pair) { HandleFramePair(pair); };
-}
-
-void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) {
- double psnr = NAN;
- RTC_CHECK(sample.captured);
- ++stats_.captures_count;
- if (!sample.decoded) {
- ++stats_.lost_count;
- } else {
- psnr = I420PSNR(*sample.captured->ToI420(), *sample.decoded->ToI420());
- ++stats_.valid_count;
- stats_.end_to_end_seconds.AddSample(
- (sample.render_time - sample.capture_time).seconds<double>());
- stats_.psnr.AddSample(psnr);
- }
- if (writer_) {
- LogWriteFormat(writer_.get(), "%.3f %.3f %.3f %i %i %i %i %.3f\n",
- sample.capture_time.seconds<double>(),
- sample.render_time.seconds<double>(),
- sample.captured->width(), sample.captured->height(),
- sample.decoded->width(), sample.decoded->height(), psnr);
- }
-}
-
-VideoQualityStats VideoQualityAnalyzer::stats() const {
- return stats_;
-}
-
} // namespace test
} // namespace webrtc
diff --git a/test/scenario/quality_stats.h b/test/scenario/video_frame_matcher.h
similarity index 67%
rename from test/scenario/quality_stats.h
rename to test/scenario/video_frame_matcher.h
index 26dd6a3..1a4a60a 100644
--- a/test/scenario/quality_stats.h
+++ b/test/scenario/video_frame_matcher.h
@@ -7,8 +7,8 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef TEST_SCENARIO_QUALITY_STATS_H_
-#define TEST_SCENARIO_QUALITY_STATS_H_
+#ifndef TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_
+#define TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_
#include <deque>
#include <map>
@@ -17,19 +17,14 @@
#include <string>
#include <vector>
-#include "absl/types/optional.h"
#include "api/units/timestamp.h"
#include "api/video/video_frame.h"
#include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h"
#include "rtc_base/ref_counted_object.h"
#include "rtc_base/task_queue_for_test.h"
-#include "rtc_base/time_utils.h"
#include "system_wrappers/include/clock.h"
-#include "test/logging/log_writer.h"
-#include "test/scenario/quality_info.h"
-#include "test/scenario/scenario_config.h"
-#include "test/statistics.h"
+#include "test/scenario/performance_stats.h"
namespace webrtc {
namespace test {
@@ -72,21 +67,7 @@
rtc::scoped_refptr<DecodedFrame> last_decode;
int next_decoded_id = 1;
};
- void HandleMatch(CapturedFrame& captured, int layer_id) {
- VideoFramePair frame_pair;
- frame_pair.layer_id = layer_id;
- frame_pair.captured = captured.frame;
- frame_pair.capture_id = captured.id;
- if (captured.best_decode) {
- frame_pair.decode_id = captured.best_decode->id;
- frame_pair.capture_time = captured.capture_time;
- frame_pair.decoded = captured.best_decode->frame;
- frame_pair.render_time = captured.best_decode->render_time;
- frame_pair.repeated = captured.best_decode->repeat_count++;
- }
- for (auto& handler : frame_pair_handlers_)
- handler(frame_pair);
- }
+ void HandleMatch(CapturedFrame captured, int layer_id);
void Finalize();
int next_capture_id_ = 1;
std::vector<std::function<void(const VideoFramePair&)>> frame_pair_handlers_;
@@ -133,35 +114,6 @@
VideoFrameMatcher* const matcher_;
int layer_id_;
};
-struct VideoQualityAnalyzerConfig {
- double psnr_coverage = 1;
-};
-struct VideoQualityStats {
- int captures_count = 0;
- int valid_count = 0;
- int lost_count = 0;
- Statistics end_to_end_seconds;
- Statistics frame_size;
- Statistics psnr;
-};
-
-class VideoQualityAnalyzer {
- public:
- explicit VideoQualityAnalyzer(
- VideoQualityAnalyzerConfig config = VideoQualityAnalyzerConfig(),
- std::unique_ptr<RtcEventLogOutput> writer = nullptr);
- ~VideoQualityAnalyzer();
- void HandleFramePair(VideoFramePair sample);
- VideoQualityStats stats() const;
- void PrintHeaders();
- void PrintFrameInfo(const VideoFramePair& sample);
- std::function<void(const VideoFramePair&)> Handler();
-
- private:
- const VideoQualityAnalyzerConfig config_;
- VideoQualityStats stats_;
- const std::unique_ptr<RtcEventLogOutput> writer_;
-};
} // namespace test
} // namespace webrtc
-#endif // TEST_SCENARIO_QUALITY_STATS_H_
+#endif // TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_
diff --git a/test/scenario/video_stream.h b/test/scenario/video_stream.h
index 66e8e90..cac5861 100644
--- a/test/scenario/video_stream.h
+++ b/test/scenario/video_stream.h
@@ -21,8 +21,8 @@
#include "test/scenario/call_client.h"
#include "test/scenario/column_printer.h"
#include "test/scenario/network_node.h"
-#include "test/scenario/quality_stats.h"
#include "test/scenario/scenario_config.h"
+#include "test/scenario/video_frame_matcher.h"
#include "test/test_video_capturer.h"
namespace webrtc {
