Introduce default video quality analyzer
This implementation won't support spatial layers and simulcast. It will
be added in next CLs.
Bug: webrtc:10138
Change-Id: I08baef36fb15b8d2d2fa222c761d40508de7ff61
Reviewed-on: https://webrtc-review.googlesource.com/c/121944
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Peter Slatala <psla@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26676}diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn
index 11293b8..27955a3 100644
--- a/test/pc/e2e/BUILD.gn
+++ b/test/pc/e2e/BUILD.gn
@@ -261,7 +261,7 @@
"peer_connection_e2e_smoke_test.cc",
]
deps = [
- ":example_video_quality_analyzer",
+ ":default_video_quality_analyzer",
"../../../api:callfactory_api",
"../../../api:libjingle_peerconnection_api",
"../../../api:scoped_refptr",
@@ -313,3 +313,34 @@
"api:video_quality_analyzer_api",
]
}
+
+rtc_source_set("default_video_quality_analyzer") {
+ visibility = [ "*" ]
+ testonly = true
+ sources = [
+ "analyzer/video/default_video_quality_analyzer.cc",
+ "analyzer/video/default_video_quality_analyzer.h",
+ ]
+
+ deps = [
+ "../..:perf_test",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../api/video:encoded_image",
+ "../../../api/video:video_frame",
+ "../../../common_video:common_video",
+ "../../../rtc_base:criticalsection",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:rtc_base_approved",
+ "../../../rtc_base:rtc_event",
+ "../../../rtc_base:rtc_numerics",
+ "../../../system_wrappers:system_wrappers",
+ "api:video_quality_analyzer_api",
+ "//third_party/abseil-cpp/absl/memory:memory",
+ ]
+
+ if (!build_with_chromium && is_clang) {
+ # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
+ suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
+ }
+}
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
new file mode 100644
index 0000000..9828540
--- /dev/null
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -0,0 +1,542 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "api/units/time_delta.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_base/logging.h"
+#include "test/testsupport/perf_test.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+constexpr int kMaxActiveComparisons = 10;
+constexpr int kFreezeThresholdMs = 150;
+
+} // namespace
+
+void RateCounter::AddEvent(Timestamp event_time) {
+ if (event_first_time_.IsMinusInfinity()) {
+ event_first_time_ = event_time;
+ }
+ event_last_time_ = event_time;
+ event_count_++;
+}
+
+double RateCounter::GetEventsPerSecond() const {
+ RTC_DCHECK(!IsEmpty());
+ return static_cast<double>(event_count_) /
+ (event_last_time_ - event_first_time_).seconds();
+}
+
+DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(std::string test_label)
+ : test_label_(std::move(test_label)), clock_(Clock::GetRealTimeClock()) {}
+DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
+ Stop();
+}
+
+void DefaultVideoQualityAnalyzer::Start(int max_threads_count) {
+ for (int i = 0; i < max_threads_count; i++) {
+ auto thread = absl::make_unique<rtc::PlatformThread>(
+ &DefaultVideoQualityAnalyzer::ProcessComparisonsThread, this,
+ ("DefaultVideoQualityAnalyzerWorker-" + std::to_string(i)).data(),
+ rtc::ThreadPriority::kNormalPriority);
+ thread->Start();
+ thread_pool_.push_back(std::move(thread));
+ }
+ {
+ rtc::CritScope crit(&lock_);
+ state_ = State::kActive;
+ }
+}
+
+uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
+ const std::string& stream_label,
+ const webrtc::VideoFrame& frame) {
+ // |next_frame_id| is atomic, so we needn't lock here.
+ uint16_t frame_id = next_frame_id_++;
+ {
+ // Ensure stats for this stream exists.
+ rtc::CritScope crit(&comparison_lock_);
+ if (stream_stats_.find(stream_label) == stream_stats_.end()) {
+ stream_stats_.insert({stream_label, StreamStats()});
+ // Assume that the first freeze was before first stream frame captured.
+ // This way time before the first freeze would be counted as time between
+ // freezes.
+ stream_last_freeze_end_time_.insert({stream_label, Now()});
+ }
+ }
+ {
+ rtc::CritScope crit(&lock_);
+ frame_counters_.captured++;
+ stream_frame_counters_[stream_label].captured++;
+
+ StreamState* state = &stream_states_[stream_label];
+ state->frame_ids.push_back(frame_id);
+ // Update frames in flight info.
+ auto it = captured_frames_in_flight_.find(frame_id);
+ if (it != captured_frames_in_flight_.end()) {
+ // We overflow uint16_t and hit previous frame id and this frame is still
+ // in flight. It means that this stream wasn't rendered for long time and
+ // we need to process existing frame as dropped.
+ auto stats_it = frame_stats_.find(frame_id);
+ RTC_DCHECK(stats_it != frame_stats_.end());
+
+ RTC_DCHECK(frame_id == state->frame_ids.front());
+ state->frame_ids.pop_front();
+ frame_counters_.dropped++;
+ stream_frame_counters_[stream_label].dropped++;
+ AddComparison(it->second, state->last_rendered_frame, true,
+ stats_it->second);
+
+ captured_frames_in_flight_.erase(it);
+ frame_stats_.erase(stats_it);
+ }
+ captured_frames_in_flight_.insert(
+ std::pair<uint16_t, VideoFrame>(frame_id, frame));
+ // Set frame id on local copy of the frame
+ captured_frames_in_flight_.at(frame_id).set_id(frame_id);
+ frame_stats_.insert(std::pair<uint16_t, FrameStats>(
+ frame_id, FrameStats(stream_label, /*captured_time=*/Now())));
+ }
+ return frame_id;
+}
+
+void DefaultVideoQualityAnalyzer::OnFramePreEncode(
+ const webrtc::VideoFrame& frame) {
+ rtc::CritScope crit(&lock_);
+ auto it = frame_stats_.find(frame.id());
+ RTC_DCHECK(it != frame_stats_.end());
+ frame_counters_.pre_encoded++;
+ stream_frame_counters_[it->second.stream_label].pre_encoded++;
+ it->second.pre_encode_time = Now();
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameEncoded(
+ uint16_t frame_id,
+ const webrtc::EncodedImage& encoded_image) {
+ rtc::CritScope crit(&lock_);
+ // TODO(titovartem) we need to pick right spatial index here.
+ auto it = frame_stats_.find(frame_id);
+ RTC_DCHECK(it != frame_stats_.end());
+ RTC_DCHECK(it->second.encoded_time.IsInfinite())
+ << "Received multiple spatial layers for stream_label="
+ << it->second.stream_label;
+ frame_counters_.encoded++;
+ stream_frame_counters_[it->second.stream_label].encoded++;
+ it->second.encoded_time = Now();
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameDropped(
+ webrtc::EncodedImageCallback::DropReason reason) {
+ // Here we do nothing, because we will see this drop on renderer side.
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameReceived(
+ uint16_t frame_id,
+ const webrtc::EncodedImage& input_image) {
+ // TODO(titovartem) We should always receive only single spatial layer here.
+ rtc::CritScope crit(&lock_);
+ auto it = frame_stats_.find(frame_id);
+ RTC_DCHECK(it != frame_stats_.end());
+ RTC_DCHECK(it->second.received_time.IsInfinite())
+ << "Received multiple spatial layers for stream_label="
+ << it->second.stream_label;
+ frame_counters_.received++;
+ stream_frame_counters_[it->second.stream_label].received++;
+ it->second.received_time = Now();
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameDecoded(
+ const webrtc::VideoFrame& frame,
+ absl::optional<int32_t> decode_time_ms,
+ absl::optional<uint8_t> qp) {
+ rtc::CritScope crit(&lock_);
+ auto it = frame_stats_.find(frame.id());
+ RTC_DCHECK(it != frame_stats_.end());
+ frame_counters_.decoded++;
+ stream_frame_counters_[it->second.stream_label].decoded++;
+ it->second.decoded_time = Now();
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameRendered(
+ const webrtc::VideoFrame& frame) {
+ rtc::CritScope crit(&lock_);
+ auto stats_it = frame_stats_.find(frame.id());
+ RTC_DCHECK(stats_it != frame_stats_.end());
+ FrameStats* frame_stats = &stats_it->second;
+ // Update frames counters.
+ frame_counters_.rendered++;
+ stream_frame_counters_[frame_stats->stream_label].rendered++;
+
+ // Update current frame stats.
+ frame_stats->rendered_time = Now();
+ frame_stats->rendered_frame_width = frame.width();
+ frame_stats->rendered_frame_height = frame.height();
+
+ // Find corresponding captured frame.
+ auto frame_it = captured_frames_in_flight_.find(frame.id());
+ RTC_DCHECK(frame_it != captured_frames_in_flight_.end());
+ const VideoFrame& captured_frame = frame_it->second;
+
+ // After we received frame here we need to check if there are any dropped
+ // frames between this one and last one, that was rendered for this video
+ // stream.
+
+ const std::string& stream_label = frame_stats->stream_label;
+ StreamState* state = &stream_states_[stream_label];
+ int dropped_count = 0;
+ while (!state->frame_ids.empty() && state->frame_ids.front() != frame.id()) {
+ dropped_count++;
+ uint16_t dropped_frame_id = state->frame_ids.front();
+ state->frame_ids.pop_front();
+ // Frame with id |dropped_frame_id| was dropped. We need:
+ // 1. Update global and stream frame counters
+ // 2. Extract corresponding frame from |captured_frames_in_flight_|
+ // 3. Extract corresponding frame stats from |frame_stats_|
+ // 4. Send extracted frame to comparison with dropped=true
+ // 5. Cleanup dropped frame
+ frame_counters_.dropped++;
+ stream_frame_counters_[stream_label].dropped++;
+
+ auto dropped_frame_stats_it = frame_stats_.find(dropped_frame_id);
+ RTC_DCHECK(dropped_frame_stats_it != frame_stats_.end());
+ auto dropped_frame_it = captured_frames_in_flight_.find(dropped_frame_id);
+ RTC_CHECK(dropped_frame_it != captured_frames_in_flight_.end());
+
+ AddComparison(dropped_frame_it->second, state->last_rendered_frame, true,
+ dropped_frame_stats_it->second);
+
+ frame_stats_.erase(dropped_frame_stats_it);
+ captured_frames_in_flight_.erase(dropped_frame_it);
+ }
+ RTC_DCHECK(!state->frame_ids.empty());
+ state->frame_ids.pop_front();
+
+ state->last_rendered_frame = frame;
+ if (state->last_rendered_frame_time) {
+ frame_stats->prev_frame_rendered_time =
+ state->last_rendered_frame_time.value();
+ }
+ state->last_rendered_frame_time = frame_stats->rendered_time;
+ {
+ rtc::CritScope cr(&comparison_lock_);
+ stream_stats_[stream_label].skipped_between_rendered.AddSample(
+ dropped_count);
+ }
+ AddComparison(captured_frame, frame, false, *frame_stats);
+
+ captured_frames_in_flight_.erase(frame_it);
+ frame_stats_.erase(stats_it);
+}
+
+void DefaultVideoQualityAnalyzer::OnEncoderError(
+ const webrtc::VideoFrame& frame,
+ int32_t error_code) {
+ RTC_LOG(LS_ERROR) << "Encoder error for frame.id=" << frame.id()
+ << ", code=" << error_code;
+}
+
+void DefaultVideoQualityAnalyzer::OnDecoderError(uint16_t frame_id,
+ int32_t error_code) {
+ RTC_LOG(LS_ERROR) << "Decoder error for frame_id=" << frame_id
+ << ", code=" << error_code;
+}
+
+void DefaultVideoQualityAnalyzer::Stop() {
+ {
+ rtc::CritScope crit(&lock_);
+ if (state_ == State::kStopped) {
+ return;
+ }
+ state_ = State::kStopped;
+ }
+ comparison_available_event_.Set();
+ for (auto& thread : thread_pool_) {
+ thread->Stop();
+ }
+ // PlatformThread have to be deleted on the same thread, where it was created
+ thread_pool_.clear();
+
+ // Perform final Metrics update. On this place analyzer is stopped and no one
+ // holds any locks.
+ {
+ // Time between freezes.
+ // Count time since the last freeze to the end of the call as time
+ // between freezes.
+ rtc::CritScope crit1(&lock_);
+ rtc::CritScope crit2(&comparison_lock_);
+ for (auto& item : stream_stats_) {
+ if (item.second.freeze_time_ms.IsEmpty()) {
+ continue;
+ }
+ const StreamState& state = stream_states_[item.first];
+ if (state.last_rendered_frame_time) {
+ item.second.time_between_freezes_ms.AddSample(
+ (state.last_rendered_frame_time.value() -
+ stream_last_freeze_end_time_.at(item.first))
+ .ms());
+ }
+ }
+ }
+ ReportResults();
+}
+
+std::set<std::string> DefaultVideoQualityAnalyzer::GetKnownVideoStreams()
+ const {
+ rtc::CritScope crit2(&comparison_lock_);
+ std::set<std::string> out;
+ for (auto& item : stream_stats_) {
+ out.insert(item.first);
+ }
+ return out;
+}
+
+const FrameCounters& DefaultVideoQualityAnalyzer::GetGlobalCounters() {
+ rtc::CritScope crit(&lock_);
+ return frame_counters_;
+}
+
+const std::map<std::string, FrameCounters>&
+DefaultVideoQualityAnalyzer::GetPerStreamCounters() const {
+ rtc::CritScope crit(&lock_);
+ return stream_frame_counters_;
+}
+
+const std::map<std::string, StreamStats>&
+DefaultVideoQualityAnalyzer::GetStats() const {
+ rtc::CritScope cri(&comparison_lock_);
+ return stream_stats_;
+}
+
+const AnalyzerStats& DefaultVideoQualityAnalyzer::GetAnalyzerStats() const {
+ rtc::CritScope crit(&comparison_lock_);
+ return analyzer_stats_;
+}
+
+void DefaultVideoQualityAnalyzer::AddComparison(
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats) {
+ rtc::CritScope crit(&comparison_lock_);
+ analyzer_stats_.comparisons_queue_size.AddSample(comparisons_.size());
+ // If there too many computations waiting in the queue, we won't provide
+ // frames itself to make future computations lighter.
+ if (comparisons_.size() >= kMaxActiveComparisons) {
+ comparisons_.emplace_back(dropped, frame_stats);
+ } else {
+ comparisons_.emplace_back(std::move(captured), std::move(rendered), dropped,
+ frame_stats);
+ }
+ comparison_available_event_.Set();
+}
+
+void DefaultVideoQualityAnalyzer::ProcessComparisonsThread(void* obj) {
+ static_cast<DefaultVideoQualityAnalyzer*>(obj)->ProcessComparisons();
+}
+
+void DefaultVideoQualityAnalyzer::ProcessComparisons() {
+ while (true) {
+ // Try to pick next comparison to perform from the queue.
+ absl::optional<FrameComparison> comparison = absl::nullopt;
+ {
+ rtc::CritScope crit(&comparison_lock_);
+ if (!comparisons_.empty()) {
+ comparison = comparisons_.front();
+ comparisons_.pop_front();
+ if (!comparisons_.empty()) {
+ comparison_available_event_.Set();
+ }
+ }
+ }
+ if (!comparison) {
+ bool more_frames_expected;
+ {
+ // If there are no comparisons and state is stopped =>
+ // no more frames expected.
+ rtc::CritScope crit(&lock_);
+ more_frames_expected = state_ != State::kStopped;
+ }
+ if (!more_frames_expected) {
+ comparison_available_event_.Set();
+ return;
+ }
+ comparison_available_event_.Wait(1000);
+ continue;
+ }
+
+ ProcessComparison(comparison.value());
+ }
+}
+
+void DefaultVideoQualityAnalyzer::ProcessComparison(
+ const FrameComparison& comparison) {
+ // Perform expensive psnr and ssim calculations while not holding lock.
+ double psnr = -1.0;
+ double ssim = -1.0;
+ if (comparison.captured && !comparison.dropped) {
+ psnr = I420PSNR(&*comparison.captured, &*comparison.rendered);
+ ssim = I420SSIM(&*comparison.captured, &*comparison.rendered);
+ }
+
+ const FrameStats& frame_stats = comparison.frame_stats;
+
+ rtc::CritScope crit(&comparison_lock_);
+ auto stats_it = stream_stats_.find(frame_stats.stream_label);
+ RTC_CHECK(stats_it != stream_stats_.end());
+ StreamStats* stats = &stats_it->second;
+ analyzer_stats_.comparisons_done++;
+ if (!comparison.captured) {
+ analyzer_stats_.overloaded_comparisons_done++;
+ }
+ if (psnr > 0) {
+ stats->psnr.AddSample(psnr);
+ }
+ if (ssim > 0) {
+ stats->ssim.AddSample(ssim);
+ }
+ if (frame_stats.encoded_time.IsFinite()) {
+ stats->encode_time_ms.AddSample(
+ (frame_stats.encoded_time - frame_stats.pre_encode_time).ms());
+ stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
+ } else {
+ if (frame_stats.pre_encode_time.IsFinite()) {
+ stats->dropped_by_encoder++;
+ } else {
+ stats->dropped_before_encoder++;
+ }
+ }
+ // Next stats can be calculated only if frame was received on remote side.
+ if (!comparison.dropped) {
+ stats->resolution_of_encoded_image.AddSample(
+ *comparison.frame_stats.rendered_frame_width *
+ *comparison.frame_stats.rendered_frame_height);
+ stats->transport_time_ms.AddSample(
+ (frame_stats.received_time - frame_stats.encoded_time).ms());
+ stats->total_delay_incl_transport_ms.AddSample(
+ (frame_stats.rendered_time - frame_stats.captured_time).ms());
+ stats->decode_time_ms.AddSample(
+ (frame_stats.decoded_time - frame_stats.received_time).ms());
+
+ if (frame_stats.prev_frame_rendered_time.IsFinite()) {
+ TimeDelta time_between_rendered_frames =
+ frame_stats.rendered_time - frame_stats.prev_frame_rendered_time;
+ stats->time_between_rendered_frames_ms.AddSample(
+ time_between_rendered_frames.ms());
+ double average_time_between_rendered_frames_ms =
+ stats->time_between_rendered_frames_ms.GetAverage();
+ if (time_between_rendered_frames.ms() >
+ std::max(kFreezeThresholdMs + average_time_between_rendered_frames_ms,
+ 3 * average_time_between_rendered_frames_ms)) {
+ stats->freeze_time_ms.AddSample(time_between_rendered_frames.ms());
+ auto freeze_end_it =
+ stream_last_freeze_end_time_.find(frame_stats.stream_label);
+ RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end());
+ stats->time_between_freezes_ms.AddSample(
+ (frame_stats.prev_frame_rendered_time - freeze_end_it->second)
+ .ms());
+ freeze_end_it->second = frame_stats.rendered_time;
+ }
+ }
+ }
+}
+
+void DefaultVideoQualityAnalyzer::ReportResults() const {
+ rtc::CritScope crit1(&lock_);
+ rtc::CritScope crit2(&comparison_lock_);
+ for (auto& item : stream_stats_) {
+ ReportResults(GetTestCaseName(item.first), item.second,
+ stream_frame_counters_.at(item.first));
+ }
+}
+
+void DefaultVideoQualityAnalyzer::ReportResults(std::string test_case_name,
+ StreamStats stats,
+ FrameCounters frame_counters) {
+ ReportResult("psnr", test_case_name, stats.psnr, "dB");
+ ReportResult("ssim", test_case_name, stats.ssim, "unitless");
+ ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms");
+ ReportResult("total_delay_incl_transport", test_case_name,
+ stats.total_delay_incl_transport_ms, "ms");
+ ReportResult("time_between_rendered_frames", test_case_name,
+ stats.time_between_rendered_frames_ms, "ms");
+ test::PrintResult("encode_frame_rate", "", test_case_name,
+ stats.encode_frame_rate.IsEmpty()
+ ? 0
+ : stats.encode_frame_rate.GetEventsPerSecond(),
+ "fps", /*important=*/false);
+ ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms");
+ ReportResult("time_between_freezes", test_case_name,
+ stats.time_between_freezes_ms, "ms");
+ ReportResult("pixels_per_frame", test_case_name,
+ stats.resolution_of_encoded_image, "unitless");
+ test::PrintResult("min_psnr", "", test_case_name,
+ stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
+ /*important=*/false);
+ ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms");
+ test::PrintResult("dropped_frames", "", test_case_name,
+ frame_counters.dropped, "unitless",
+ /*important=*/false);
+ ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
+ "unitless");
+}
+
+void DefaultVideoQualityAnalyzer::ReportResult(
+ const std::string& metric_name,
+ const std::string& test_case_name,
+ const SamplesStatsCounter& counter,
+ const std::string& unit) {
+ test::PrintResultMeanAndError(
+ metric_name, /*modifier=*/"", test_case_name,
+ counter.IsEmpty() ? 0 : counter.GetAverage(),
+ counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit,
+ /*important=*/false);
+}
+
+std::string DefaultVideoQualityAnalyzer::GetTestCaseName(
+ const std::string& stream_label) const {
+ return test_label_ + "/" + stream_label;
+}
+
+Timestamp DefaultVideoQualityAnalyzer::Now() {
+ return Timestamp::us(clock_->TimeInMicroseconds());
+}
+
+DefaultVideoQualityAnalyzer::FrameStats::FrameStats(std::string stream_label,
+ Timestamp captured_time)
+ : stream_label(std::move(stream_label)), captured_time(captured_time) {}
+
+DefaultVideoQualityAnalyzer::FrameComparison::FrameComparison(
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats)
+ : captured(std::move(captured)),
+ rendered(std::move(rendered)),
+ dropped(dropped),
+ frame_stats(std::move(frame_stats)) {}
+
+DefaultVideoQualityAnalyzer::FrameComparison::FrameComparison(
+ bool dropped,
+ FrameStats frame_stats)
+ : captured(absl::nullopt),
+ rendered(absl::nullopt),
+ dropped(dropped),
+ frame_stats(std::move(frame_stats)) {}
+
+} // namespace test
+} // namespace webrtc
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
new file mode 100644
index 0000000..2e28829
--- /dev/null
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
@@ -0,0 +1,280 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_
+
+#include <atomic>
+#include <deque>
+#include <list>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "api/units/timestamp.h"
+#include "api/video/encoded_image.h"
+#include "api/video/video_frame.h"
+#include "rtc_base/critical_section.h"
+#include "rtc_base/event.h"
+#include "rtc_base/numerics/samples_stats_counter.h"
+#include "rtc_base/platform_thread.h"
+#include "system_wrappers/include/clock.h"
+#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
+
+namespace webrtc {
+namespace test {
+
+class RateCounter {
+ public:
+ void AddEvent(Timestamp event_time);
+
+ bool IsEmpty() const { return event_first_time_ == event_last_time_; }
+
+ double GetEventsPerSecond() const;
+
+ private:
+ Timestamp event_first_time_ = Timestamp::MinusInfinity();
+ Timestamp event_last_time_ = Timestamp::MinusInfinity();
+ int64_t event_count_ = 0;
+};
+
+struct FrameCounters {
+ // Count of frames, that were passed into WebRTC pipeline by video stream
+ // source.
+ int64_t captured = 0;
+ // Count of frames that reached video encoder.
+ int64_t pre_encoded = 0;
+ // Count of encoded images that were produced by encoder for all requested
+ // spatial layers and simulcast streams.
+ int64_t encoded = 0;
+ // Count of encoded images received in decoder for all requested spatial
+ // layers and simulcast streams.
+ int64_t received = 0;
+ // Count of frames that were produced by decoder.
+ int64_t decoded = 0;
+ // Count of frames that went out from WebRTC pipeline to video sink.
+ int64_t rendered = 0;
+ // Count of frames that were dropped in any point between capturing and
+ // rendering.
+ int64_t dropped = 0;
+};
+
+struct StreamStats {
+ public:
+ SamplesStatsCounter psnr;
+ SamplesStatsCounter ssim;
+ // Time from frame encoded (time point on exit from encoder) to the
+ // encoded image received in decoder (time point on entrance to decoder).
+ SamplesStatsCounter transport_time_ms;
+ // Time from frame was captured on device to time frame was displayed on
+ // device.
+ SamplesStatsCounter total_delay_incl_transport_ms;
+ // Time between frames out from renderer.
+ SamplesStatsCounter time_between_rendered_frames_ms;
+ RateCounter encode_frame_rate;
+ SamplesStatsCounter encode_time_ms;
+ SamplesStatsCounter decode_time_ms;
+ // Max frames skipped between two nearest.
+ SamplesStatsCounter skipped_between_rendered;
+ // In the next 2 metrics freeze is a pause that is longer, than maximum:
+ // 1. 150ms
+ // 2. 3 * average time between two sequential frames.
+ // Item 1 will cover high fps video and is a duration, that is noticeable by
+ // human eye. Item 2 will cover low fps video like screen sharing.
+ // Freeze duration.
+ SamplesStatsCounter freeze_time_ms;
+ // Mean time between one freeze end and next freeze start.
+ SamplesStatsCounter time_between_freezes_ms;
+ SamplesStatsCounter resolution_of_encoded_image;
+
+ int64_t dropped_by_encoder = 0;
+ int64_t dropped_before_encoder = 0;
+};
+
+struct AnalyzerStats {
+ public:
+ // Size of analyzer internal comparisons queue, measured when new element
+ // id added to the queue.
+ SamplesStatsCounter comparisons_queue_size;
+ // Amount of performed comparisons of 2 video frames from captured and
+ // rendered streams.
+ int64_t comparisons_done = 0;
+ // Amount of overloaded comparisons. Comparison is overloaded if it is queued
+ // when there are too many not processed comparisons in the queue. Overloaded
+ // comparison doesn't include metrics, that require heavy computations like
+ // SSIM and PSNR.
+ int64_t overloaded_comparisons_done = 0;
+};
+
+class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
+ public:
+ explicit DefaultVideoQualityAnalyzer(std::string test_label);
+ ~DefaultVideoQualityAnalyzer() override;
+
+ void Start(int max_threads_count) override;
+ uint16_t OnFrameCaptured(const std::string& stream_label,
+ const VideoFrame& frame) override;
+ void OnFramePreEncode(const VideoFrame& frame) override;
+ void OnFrameEncoded(uint16_t frame_id,
+ const EncodedImage& encoded_image) override;
+ void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
+ void OnFrameReceived(uint16_t frame_id,
+ const EncodedImage& input_image) override;
+ void OnFrameDecoded(const VideoFrame& frame,
+ absl::optional<int32_t> decode_time_ms,
+ absl::optional<uint8_t> qp) override;
+ void OnFrameRendered(const VideoFrame& frame) override;
+ void OnEncoderError(const VideoFrame& frame, int32_t error_code) override;
+ void OnDecoderError(uint16_t frame_id, int32_t error_code) override;
+ void Stop() override;
+
+ // Returns set of stream labels, that were met during test call.
+ std::set<std::string> GetKnownVideoStreams() const;
+ const FrameCounters& GetGlobalCounters();
+ // Returns frame counter per stream label. Valid stream labels can be obtained
+ // by calling GetKnownVideoStreams()
+ const std::map<std::string, FrameCounters>& GetPerStreamCounters() const;
+ // Returns video quality stats per stream label. Valid stream labels can be
+ // obtained by calling GetKnownVideoStreams()
+ const std::map<std::string, StreamStats>& GetStats() const;
+ const AnalyzerStats& GetAnalyzerStats() const;
+
+ private:
+ struct FrameStats {
+ FrameStats(std::string stream_label, Timestamp captured_time);
+
+ std::string stream_label;
+
+ // Frame events timestamp.
+ Timestamp captured_time;
+ Timestamp pre_encode_time = Timestamp::MinusInfinity();
+ Timestamp encoded_time = Timestamp::MinusInfinity();
+ Timestamp received_time = Timestamp::MinusInfinity();
+ Timestamp decoded_time = Timestamp::MinusInfinity();
+ Timestamp rendered_time = Timestamp::MinusInfinity();
+ Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
+
+ absl::optional<int> rendered_frame_width = absl::nullopt;
+ absl::optional<int> rendered_frame_height = absl::nullopt;
+ };
+
+ // Represents comparison between two VideoFrames. Contains video frames itself
+ // and stats. Can be one of two types:
+ // 1. Normal - in this case |captured| is presented and either |rendered| is
+ // presented and |dropped| is false, either |rendered| is omitted and
+ // |dropped| is true.
+ // 2. Overloaded - in this case both |captured| and |rendered| are omitted
+ // because there were too many comparisons in the queue. |dropped| can be
+ // true or false showing was frame dropped or not.
+ struct FrameComparison {
+ FrameComparison(absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats);
+ FrameComparison(bool dropped, FrameStats frameStats);
+
+ // Frames can be omitted if there too many computations waiting in the
+ // queue.
+ absl::optional<VideoFrame> captured;
+ absl::optional<VideoFrame> rendered;
+ // If true frame was dropped somewhere from capturing to rendering and
+ // wasn't rendered on remote peer side. If |dropped| is true, |rendered|
+ // will be |absl::nullopt|.
+ bool dropped;
+ FrameStats frame_stats;
+ };
+
+ // Represents a current state of video stream.
+ struct StreamState {
+ // To correctly determine dropped frames we have to know sequence of frames
+ // in each stream so we will keep a list of frame ids inside the stream.
+ // When the frame is rendered, we will pop ids from the list for until id
+ // will match with rendered one. All ids before matched one can be
+ // considered as dropped:
+ //
+ // | frame_id1 |->| frame_id2 |->| frame_id3 |->| frame_id4 |
+ //
+ // If we received frame with id frame_id3, then we will pop frame_id1 and
+ // frame_id2 and consider that frames as dropped and then compare received
+ // frame with the one from |captured_frames_in_flight_| with id frame_id3.
+ // Also we will put it into the |last_rendered_frame|.
+ std::list<uint16_t> frame_ids;
+ absl::optional<VideoFrame> last_rendered_frame = absl::nullopt;
+ absl::optional<Timestamp> last_rendered_frame_time = absl::nullopt;
+ };
+
+ enum State { kNew, kActive, kStopped };
+
+ // Returns last rendered frame for stream if there is one or nullptr
+ // otherwise.
+ VideoFrame* GetLastRenderedFrame(const std::string& stream_label)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
+ void SetLastRenderedFrame(const std::string& stream_label,
+ const VideoFrame& frame)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
+
+ void AddComparison(absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats);
+ static void ProcessComparisonsThread(void* obj);
+ void ProcessComparisons();
+ void ProcessComparison(const FrameComparison& comparison);
+ // Report results for all metrics for all streams.
+ void ReportResults() const;
+ static void ReportResults(std::string test_case_name,
+ StreamStats stats,
+ FrameCounters frame_counters);
+ // Report result for single metric for specified stream.
+ static void ReportResult(const std::string& metric_name,
+ const std::string& test_case_name,
+ const SamplesStatsCounter& counter,
+ const std::string& unit);
+ // Returns name of current test case for reporting.
+ std::string GetTestCaseName(const std::string& stream_label) const;
+ Timestamp Now();
+
+ const std::string test_label_;
+
+ webrtc::Clock* const clock_;
+ std::atomic<uint16_t> next_frame_id_{0};
+
+ rtc::CriticalSection lock_;
+ State state_ RTC_GUARDED_BY(lock_) = State::kNew;
+ // Frames that were captured by all streams and still aren't rendered by any
+ // stream or deemed dropped.
+ std::map<uint16_t, VideoFrame> captured_frames_in_flight_
+ RTC_GUARDED_BY(lock_);
+ // Global frames count for all video streams.
+ FrameCounters frame_counters_ RTC_GUARDED_BY(lock_);
+ // Frame counters per each stream.
+ std::map<std::string, FrameCounters> stream_frame_counters_
+ RTC_GUARDED_BY(lock_);
+ std::map<uint16_t, FrameStats> frame_stats_ RTC_GUARDED_BY(lock_);
+ std::map<std::string, StreamState> stream_states_ RTC_GUARDED_BY(lock_);
+
+ rtc::CriticalSection comparison_lock_;
+ std::map<std::string, StreamStats> stream_stats_
+ RTC_GUARDED_BY(comparison_lock_);
+ std::map<std::string, Timestamp> stream_last_freeze_end_time_
+ RTC_GUARDED_BY(comparison_lock_);
+ std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
+ AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(comparison_lock_);
+
+ std::vector<std::unique_ptr<rtc::PlatformThread>> thread_pool_;
+ rtc::Event comparison_available_event_;
+};
+
+} // namespace test
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_
diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc
index a126580..0cfe3d9 100644
--- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc
+++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc
@@ -16,7 +16,7 @@
#include "rtc_base/async_invoker.h"
#include "rtc_base/fake_network.h"
#include "test/gtest.h"
-#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
#include "test/pc/e2e/api/create_peerconnection_quality_test_fixture.h"
#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h"
#include "test/scenario/network/network_emulation.h"
@@ -101,8 +101,8 @@
// Create analyzers.
auto analyzers = absl::make_unique<Analyzers>();
analyzers->video_quality_analyzer =
- absl::make_unique<ExampleVideoQualityAnalyzer>();
- auto* video_analyzer = static_cast<ExampleVideoQualityAnalyzer*>(
+ absl::make_unique<DefaultVideoQualityAnalyzer>("smoke_test");
+ auto* video_analyzer = static_cast<DefaultVideoQualityAnalyzer*>(
analyzers->video_quality_analyzer.get());
auto fixture =
@@ -111,17 +111,16 @@
std::move(bob_components), absl::make_unique<Params>(),
RunParams{TimeDelta::seconds(5)});
- RTC_LOG(INFO) << "Captured: " << video_analyzer->frames_captured();
- RTC_LOG(INFO) << "Sent : " << video_analyzer->frames_sent();
- RTC_LOG(INFO) << "Received: " << video_analyzer->frames_received();
- RTC_LOG(INFO) << "Rendered: " << video_analyzer->frames_rendered();
- RTC_LOG(INFO) << "Dropped : " << video_analyzer->frames_dropped();
-
- // 150 = 30fps * 5s
- EXPECT_GE(video_analyzer->frames_captured(), 150lu);
- // EXPECT_NEAR(video_analyzer->frames_sent(), 150, 15);
- // EXPECT_NEAR(video_analyzer->frames_received(), 150, 15);
- // EXPECT_NEAR(video_analyzer->frames_rendered(), 150, 15);
+ // 150 = 30fps * 5s. On some devices pipeline can be too slow, so it can
+ // happen, that frames will stuck in the middle, so we actually can't force
+ // real constraints here, so lets just check, that at least 1 frame passed
+ // whole pipeline.
+ EXPECT_GE(video_analyzer->GetGlobalCounters().captured, 150);
+ EXPECT_GE(video_analyzer->GetGlobalCounters().pre_encoded, 1);
+ EXPECT_GE(video_analyzer->GetGlobalCounters().encoded, 1);
+ EXPECT_GE(video_analyzer->GetGlobalCounters().received, 1);
+ EXPECT_GE(video_analyzer->GetGlobalCounters().decoded, 1);
+ EXPECT_GE(video_analyzer->GetGlobalCounters().rendered, 1);
}
} // namespace test
