Extract frames comparator out from DVQA
Bug: b/196229820
Change-Id: Iaea04feadf0ed9cd734dd31e7ccca915fb7c585a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228645
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34776}
diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn
index ff3b701..67a84ac 100644
--- a/test/pc/e2e/BUILD.gn
+++ b/test/pc/e2e/BUILD.gn
@@ -34,6 +34,7 @@
testonly = true
deps = [
+ ":default_video_quality_analyzer_frames_comparator_test",
":default_video_quality_analyzer_test",
":multi_head_queue_test",
":peer_connection_e2e_smoke_test",
@@ -517,6 +518,19 @@
]
}
+ rtc_library("default_video_quality_analyzer_frames_comparator_test") {
+ testonly = true
+ sources = [ "analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc" ]
+ deps = [
+ ":default_video_quality_analyzer_internal",
+ ":default_video_quality_analyzer_shared",
+ "../..:test_support",
+ "../../../api:create_frame_generator",
+ "../../../api/units:timestamp",
+ "../../../system_wrappers",
+ ]
+ }
+
rtc_library("multi_head_queue_test") {
testonly = true
sources = [ "analyzer/video/multi_head_queue_test.cc" ]
@@ -651,25 +665,38 @@
# This target contains implementation details of DefaultVideoQualityAnalyzer,
# so headers exported by it shouldn't be used in other places.
rtc_library("default_video_quality_analyzer_internal") {
- visibility = [ ":default_video_quality_analyzer" ]
+ visibility = [
+ ":default_video_quality_analyzer",
+ ":default_video_quality_analyzer_frames_comparator_test",
+ ]
testonly = true
sources = [
"analyzer/video/default_video_quality_analyzer_cpu_measurer.cc",
"analyzer/video/default_video_quality_analyzer_cpu_measurer.h",
+ "analyzer/video/default_video_quality_analyzer_frames_comparator.cc",
+ "analyzer/video/default_video_quality_analyzer_frames_comparator.h",
"analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc",
"analyzer/video/default_video_quality_analyzer_internal_shared_objects.h",
]
deps = [
":default_video_quality_analyzer_shared",
+ "../../../api:array_view",
+ "../../../api:scoped_refptr",
"../../../api/numerics:numerics",
"../../../api/units:timestamp",
"../../../api/video:video_frame",
+ "../../../common_video",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:rtc_base_approved",
"../../../rtc_base:rtc_base_tests_utils",
+ "../../../rtc_base:rtc_event",
"../../../rtc_base:stringutils",
"../../../rtc_base:timeutils",
"../../../rtc_base/synchronization:mutex",
+ "../../../rtc_tools:video_quality_analysis",
+ "../../../system_wrappers:system_wrappers",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
index f3e2344..04fdaca 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -24,6 +24,7 @@
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/time_utils.h"
#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
@@ -31,8 +32,6 @@
namespace webrtc_pc_e2e {
namespace {
-constexpr int kMaxActiveComparisons = 10;
-constexpr int kFreezeThresholdMs = 150;
constexpr int kMicrosPerSecond = 1000000;
constexpr int kBitsInByte = 8;
@@ -108,7 +107,9 @@
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options)
- : options_(options), clock_(clock) {}
+ : options_(options),
+ clock_(clock),
+ frames_comparator_(clock, cpu_measurer_, options) {}
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop();
}
@@ -118,20 +119,15 @@
rtc::ArrayView<const std::string> peer_names,
int max_threads_count) {
test_label_ = std::move(test_case_name);
- for (int i = 0; i < max_threads_count; i++) {
- thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable(
- [this] { ProcessComparisons(); },
- "DefaultVideoQualityAnalyzerWorker-" + std::to_string(i)));
- }
+ frames_comparator_.Start(max_threads_count);
{
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
peers_ = std::make_unique<NamesCollection>(peer_names);
RTC_CHECK(start_time_.IsMinusInfinity());
state_ = State::kActive;
start_time_ = Now();
}
- cpu_measurer_.StartMeasuringCpuProcessTime();
}
uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
@@ -146,40 +142,20 @@
size_t peers_count = -1;
size_t stream_index;
{
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
// Create a local copy of `start_time_`, peer's index and total peers count
- // to access it under `comparison_lock_` without holding a `lock_`
+ // to access it without holding a `mutex_` during access to
+ // `frames_comparator_`.
start_time = start_time_;
peer_index = peers_->index(peer_name);
peers_count = peers_->size();
stream_index = streams_.AddIfAbsent(stream_label);
}
+ // Ensure stats for this stream exists.
+ frames_comparator_.EnsureStatsForStream(stream_index, peer_index, peers_count,
+ captured_time, start_time);
{
- // Ensure stats for this stream exists.
- MutexLock lock(&comparison_lock_);
- for (size_t i = 0; i < peers_count; ++i) {
- if (i == peer_index && !options_.enable_receive_own_stream) {
- continue;
- }
- InternalStatsKey stats_key(stream_index, peer_index, i);
- if (stream_stats_.find(stats_key) == stream_stats_.end()) {
- stream_stats_.insert({stats_key, StreamStats(captured_time)});
- // Assume that the first freeze was before first stream frame captured.
- // This way time before the first freeze would be counted as time
- // between freezes.
- stream_last_freeze_end_time_.insert({stats_key, start_time});
- } else {
- // When we see some `stream_label` for the first time we need to create
- // stream stats object for it and set up some states, but we need to do
- // it only once and for all receivers, so on the next frame on the same
- // `stream_label` we can be sure, that it's already done and we needn't
- // to scan though all peers again.
- break;
- }
- }
- }
- {
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
stream_to_sender_[stream_index] = peer_index;
frame_counters_.captured++;
for (size_t i = 0; i < peers_->size(); ++i) {
@@ -215,12 +191,11 @@
InternalStatsKey key(stream_index, peer_index, i);
stream_frame_counters_.at(key).dropped++;
- MutexLock lock1(&comparison_lock_);
analyzer_stats_.frames_in_flight_left_count.AddSample(
StatsSample(captured_frames_in_flight_.size(), Now()));
- AddComparison(InternalStatsKey(stream_index, peer_index, i),
- it->second.frame(), absl::nullopt, true,
- it->second.GetStatsForPeer(i));
+ frames_comparator_.AddComparison(
+ InternalStatsKey(stream_index, peer_index, i), it->second.frame(),
+ absl::nullopt, true, it->second.GetStatsForPeer(i));
}
captured_frames_in_flight_.erase(it);
@@ -260,7 +235,7 @@
void DefaultVideoQualityAnalyzer::OnFramePreEncode(
absl::string_view peer_name,
const webrtc::VideoFrame& frame) {
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
auto it = captured_frames_in_flight_.find(frame.id());
RTC_DCHECK(it != captured_frames_in_flight_.end())
<< "Frame id=" << frame.id() << " not found";
@@ -280,7 +255,7 @@
uint16_t frame_id,
const webrtc::EncodedImage& encoded_image,
const EncoderStats& stats) {
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
auto it = captured_frames_in_flight_.find(frame_id);
if (it == captured_frames_in_flight_.end()) {
RTC_LOG(WARNING)
@@ -325,7 +300,7 @@
absl::string_view peer_name,
uint16_t frame_id,
const webrtc::EncodedImage& input_image) {
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
size_t peer_index = peers_->index(peer_name);
auto it = captured_frames_in_flight_.find(frame_id);
@@ -361,7 +336,7 @@
absl::string_view peer_name,
const webrtc::VideoFrame& frame,
const DecoderStats& stats) {
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
size_t peer_index = peers_->index(peer_name);
auto it = captured_frames_in_flight_.find(frame.id());
@@ -391,7 +366,7 @@
void DefaultVideoQualityAnalyzer::OnFrameRendered(
absl::string_view peer_name,
const webrtc::VideoFrame& frame) {
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
size_t peer_index = peers_->index(peer_name);
auto frame_it = captured_frames_in_flight_.find(frame.id());
@@ -442,13 +417,11 @@
absl::optional<VideoFrame> dropped_frame = dropped_frame_it->second.frame();
dropped_frame_it->second.MarkDropped(peer_index);
- {
- MutexLock lock1(&comparison_lock_);
- analyzer_stats_.frames_in_flight_left_count.AddSample(
- StatsSample(captured_frames_in_flight_.size(), Now()));
- AddComparison(stats_key, dropped_frame, absl::nullopt, true,
- dropped_frame_it->second.GetStatsForPeer(peer_index));
- }
+ analyzer_stats_.frames_in_flight_left_count.AddSample(
+ StatsSample(captured_frames_in_flight_.size(), Now()));
+ frames_comparator_.AddComparison(
+ stats_key, dropped_frame, absl::nullopt, true,
+ dropped_frame_it->second.GetStatsForPeer(peer_index));
if (dropped_frame_it->second.HaveAllPeersReceived()) {
captured_frames_in_flight_.erase(dropped_frame_it);
@@ -463,15 +436,11 @@
}
state->SetLastRenderedFrameTime(peer_index,
frame_in_flight->rendered_time(peer_index));
- {
- MutexLock lock(&comparison_lock_);
- stream_stats_.at(stats_key).skipped_between_rendered.AddSample(
- StatsSample(dropped_count, Now()));
- analyzer_stats_.frames_in_flight_left_count.AddSample(
- StatsSample(captured_frames_in_flight_.size(), Now()));
- AddComparison(stats_key, captured_frame, frame, false,
- frame_in_flight->GetStatsForPeer(peer_index));
- }
+ analyzer_stats_.frames_in_flight_left_count.AddSample(
+ StatsSample(captured_frames_in_flight_.size(), Now()));
+ frames_comparator_.AddComparison(
+ stats_key, dropped_count, captured_frame, frame, /*dropped=*/false,
+ frame_in_flight->GetStatsForPeer(peer_index));
if (frame_it->second.HaveAllPeersReceived()) {
captured_frames_in_flight_.erase(frame_it);
@@ -495,8 +464,7 @@
void DefaultVideoQualityAnalyzer::RegisterParticipantInCall(
absl::string_view peer_name) {
- MutexLock lock1(&lock_);
- MutexLock lock2(&comparison_lock_);
+ MutexLock lock(&mutex_);
RTC_CHECK(!peers_->HasName(peer_name));
size_t new_peer_index = peers_->AddIfAbsent(peer_name);
@@ -504,6 +472,7 @@
// streams exists. Since in flight frames will be sent to the new peer
// as well. Sending stats (from this peer to others) will be added by
// DefaultVideoQualityAnalyzer::OnFrameCaptured.
+ std::vector<std::pair<InternalStatsKey, Timestamp>> stream_started_time;
for (auto& key_val : stream_to_sender_) {
size_t stream_index = key_val.first;
size_t sender_peer_index = key_val.second;
@@ -528,11 +497,11 @@
// then `counters` will be empty. In such case empty `counters` are ok.
stream_frame_counters_.insert({key, std::move(counters)});
- stream_stats_.insert(
- {key,
- StreamStats(stream_states_.at(stream_index).stream_started_time())});
- stream_last_freeze_end_time_.insert({key, start_time_});
+ stream_started_time.push_back(
+ {key, stream_states_.at(stream_index).stream_started_time()});
}
+ frames_comparator_.RegisterParticipantInCall(stream_started_time,
+ start_time_);
// Ensure, that frames states are handled correctly
// (e.g. dropped frames tracking).
for (auto& key_val : stream_states_) {
@@ -549,30 +518,19 @@
}
void DefaultVideoQualityAnalyzer::Stop() {
+ std::map<InternalStatsKey, Timestamp> last_rendered_frame_times;
{
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
if (state_ == State::kStopped) {
return;
}
state_ = State::kStopped;
- }
- cpu_measurer_.StopMeasuringCpuProcessTime();
- comparison_available_event_.Set();
- thread_pool_.clear();
- // Perform final Metrics update. On this place analyzer is stopped and no one
- // holds any locks.
- {
- // Time between freezes.
- // Count time since the last freeze to the end of the call as time
- // between freezes.
- MutexLock lock1(&lock_);
- MutexLock lock2(&comparison_lock_);
for (auto& state_entry : stream_states_) {
const size_t stream_index = state_entry.first;
const StreamState& stream_state = state_entry.second;
for (size_t i = 0; i < peers_->size(); ++i) {
- if (i == static_cast<size_t>(stream_state.owner())) {
+ if (i == stream_state.owner() && !options_.enable_receive_own_stream) {
continue;
}
@@ -580,18 +538,31 @@
// If there are no freezes in the call we have to report
// time_between_freezes_ms as call duration and in such case
- // `stream_last_freeze_end_time_` for this stream will be `start_time_`.
+ // `stream_last_freeze_end_time` for this stream will be `start_time_`.
// If there is freeze, then we need add time from last rendered frame
// to last freeze end as time between freezes.
if (stream_state.last_rendered_frame_time(i)) {
- stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(
- StatsSample(
- stream_state.last_rendered_frame_time(i).value().ms() -
- stream_last_freeze_end_time_.at(stats_key).ms(),
- Now()));
+ last_rendered_frame_times.emplace(
+ stats_key, stream_state.last_rendered_frame_time(i).value());
}
}
}
+ }
+ frames_comparator_.Stop(last_rendered_frame_times);
+
+ // Perform final Metrics update. On this place analyzer is stopped and no one
+ // holds any locks.
+ {
+ MutexLock lock(&mutex_);
+ FramesComparatorStats frames_comparator_stats =
+ frames_comparator_.frames_comparator_stats();
+ analyzer_stats_.comparisons_queue_size =
+ frames_comparator_stats.comparisons_queue_size;
+ analyzer_stats_.comparisons_done = frames_comparator_stats.comparisons_done;
+ analyzer_stats_.cpu_overloaded_comparisons_done =
+ frames_comparator_stats.cpu_overloaded_comparisons_done;
+ analyzer_stats_.memory_overloaded_comparisons_done =
+ frames_comparator_stats.memory_overloaded_comparisons_done;
analyzer_stats_.frames_in_flight_left_count.AddSample(
StatsSample(captured_frames_in_flight_.size(), Now()));
}
@@ -599,7 +570,7 @@
}
std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) {
- MutexLock lock1(&lock_);
+ MutexLock lock1(&mutex_);
auto it = captured_frames_in_flight_.find(frame_id);
if (it != captured_frames_in_flight_.end()) {
return streams_.name(it->second.stream());
@@ -615,10 +586,9 @@
}
std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const {
- MutexLock lock1(&lock_);
- MutexLock lock2(&comparison_lock_);
+ MutexLock lock(&mutex_);
std::set<StatsKey> out;
- for (auto& item : stream_stats_) {
+ for (auto& item : frames_comparator_.stream_stats()) {
RTC_LOG(INFO) << item.first.ToString() << " ==> "
<< ToStatsKey(item.first).ToString();
out.insert(ToStatsKey(item.first));
@@ -627,13 +597,13 @@
}
const FrameCounters& DefaultVideoQualityAnalyzer::GetGlobalCounters() const {
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
return frame_counters_;
}
std::map<StatsKey, FrameCounters>
DefaultVideoQualityAnalyzer::GetPerStreamCounters() const {
- MutexLock lock(&lock_);
+ MutexLock lock(&mutex_);
std::map<StatsKey, FrameCounters> out;
for (auto& item : stream_frame_counters_) {
out.emplace(ToStatsKey(item.first), item.second);
@@ -642,216 +612,31 @@
}
std::map<StatsKey, StreamStats> DefaultVideoQualityAnalyzer::GetStats() const {
- MutexLock lock1(&lock_);
- MutexLock lock2(&comparison_lock_);
+ MutexLock lock1(&mutex_);
std::map<StatsKey, StreamStats> out;
- for (auto& item : stream_stats_) {
+ for (auto& item : frames_comparator_.stream_stats()) {
out.emplace(ToStatsKey(item.first), item.second);
}
return out;
}
AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const {
- MutexLock lock(&comparison_lock_);
+ MutexLock lock(&mutex_);
return analyzer_stats_;
}
-void DefaultVideoQualityAnalyzer::AddComparison(
- InternalStatsKey stats_key,
- absl::optional<VideoFrame> captured,
- absl::optional<VideoFrame> rendered,
- bool dropped,
- FrameStats frame_stats) {
- cpu_measurer_.StartExcludingCpuThreadTime();
- analyzer_stats_.comparisons_queue_size.AddSample(
- StatsSample(comparisons_.size(), Now()));
- // If there too many computations waiting in the queue, we won't provide
- // frames itself to make future computations lighter.
- if (comparisons_.size() >= kMaxActiveComparisons) {
- comparisons_.emplace_back(std::move(stats_key), absl::nullopt,
- absl::nullopt, dropped, std::move(frame_stats),
- OverloadReason::kCpu);
- } else {
- OverloadReason overload_reason = OverloadReason::kNone;
- if (!captured && !dropped) {
- overload_reason = OverloadReason::kMemory;
- }
- comparisons_.emplace_back(std::move(stats_key), std::move(captured),
- std::move(rendered), dropped,
- std::move(frame_stats), overload_reason);
- }
- comparison_available_event_.Set();
- cpu_measurer_.StopExcludingCpuThreadTime();
-}
-
-void DefaultVideoQualityAnalyzer::ProcessComparisons() {
- while (true) {
- // Try to pick next comparison to perform from the queue.
- absl::optional<FrameComparison> comparison = absl::nullopt;
- {
- MutexLock lock(&comparison_lock_);
- if (!comparisons_.empty()) {
- comparison = comparisons_.front();
- comparisons_.pop_front();
- if (!comparisons_.empty()) {
- comparison_available_event_.Set();
- }
- }
- }
- if (!comparison) {
- bool more_frames_expected;
- {
- // If there are no comparisons and state is stopped =>
- // no more frames expected.
- MutexLock lock(&lock_);
- more_frames_expected = state_ != State::kStopped;
- }
- if (!more_frames_expected) {
- comparison_available_event_.Set();
- return;
- }
- comparison_available_event_.Wait(1000);
- continue;
- }
-
- cpu_measurer_.StartExcludingCpuThreadTime();
- ProcessComparison(comparison.value());
- cpu_measurer_.StopExcludingCpuThreadTime();
- }
-}
-
-void DefaultVideoQualityAnalyzer::ProcessComparison(
- const FrameComparison& comparison) {
- // Perform expensive psnr and ssim calculations while not holding lock.
- double psnr = -1.0;
- double ssim = -1.0;
- if (options_.heavy_metrics_computation_enabled && comparison.captured &&
- !comparison.dropped) {
- rtc::scoped_refptr<I420BufferInterface> reference_buffer =
- comparison.captured->video_frame_buffer()->ToI420();
- rtc::scoped_refptr<I420BufferInterface> test_buffer =
- comparison.rendered->video_frame_buffer()->ToI420();
- if (options_.adjust_cropping_before_comparing_frames) {
- test_buffer =
- ScaleVideoFrameBuffer(*test_buffer.get(), reference_buffer->width(),
- reference_buffer->height());
- reference_buffer = test::AdjustCropping(reference_buffer, test_buffer);
- }
- psnr = I420PSNR(*reference_buffer.get(), *test_buffer.get());
- ssim = I420SSIM(*reference_buffer.get(), *test_buffer.get());
- }
-
- const FrameStats& frame_stats = comparison.frame_stats;
-
- MutexLock lock(&comparison_lock_);
- auto stats_it = stream_stats_.find(comparison.stats_key);
- RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString();
- StreamStats* stats = &stats_it->second;
- analyzer_stats_.comparisons_done++;
- if (comparison.overload_reason == OverloadReason::kCpu) {
- analyzer_stats_.cpu_overloaded_comparisons_done++;
- } else if (comparison.overload_reason == OverloadReason::kMemory) {
- analyzer_stats_.memory_overloaded_comparisons_done++;
- }
- if (psnr > 0) {
- stats->psnr.AddSample(StatsSample(psnr, frame_stats.rendered_time));
- }
- if (ssim > 0) {
- stats->ssim.AddSample(StatsSample(ssim, frame_stats.received_time));
- }
- if (frame_stats.encoded_time.IsFinite()) {
- stats->encode_time_ms.AddSample(StatsSample(
- (frame_stats.encoded_time - frame_stats.pre_encode_time).ms(),
- frame_stats.encoded_time));
- stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
- stats->total_encoded_images_payload += frame_stats.encoded_image_size;
- stats->target_encode_bitrate.AddSample(StatsSample(
- frame_stats.target_encode_bitrate, frame_stats.encoded_time));
- } else {
- if (frame_stats.pre_encode_time.IsFinite()) {
- stats->dropped_by_encoder++;
- } else {
- stats->dropped_before_encoder++;
- }
- }
- // Next stats can be calculated only if frame was received on remote side.
- if (!comparison.dropped) {
- stats->resolution_of_rendered_frame.AddSample(
- StatsSample(*comparison.frame_stats.rendered_frame_width *
- *comparison.frame_stats.rendered_frame_height,
- frame_stats.rendered_time));
- stats->transport_time_ms.AddSample(StatsSample(
- (frame_stats.decode_start_time - frame_stats.encoded_time).ms(),
- frame_stats.received_time));
- stats->total_delay_incl_transport_ms.AddSample(StatsSample(
- (frame_stats.rendered_time - frame_stats.captured_time).ms(),
- frame_stats.received_time));
- stats->decode_time_ms.AddSample(StatsSample(
- (frame_stats.decode_end_time - frame_stats.decode_start_time).ms(),
- frame_stats.decode_end_time));
- stats->receive_to_render_time_ms.AddSample(StatsSample(
- (frame_stats.rendered_time - frame_stats.received_time).ms(),
- frame_stats.rendered_time));
-
- if (frame_stats.prev_frame_rendered_time.IsFinite()) {
- TimeDelta time_between_rendered_frames =
- frame_stats.rendered_time - frame_stats.prev_frame_rendered_time;
- stats->time_between_rendered_frames_ms.AddSample(StatsSample(
- time_between_rendered_frames.ms(), frame_stats.rendered_time));
- double average_time_between_rendered_frames_ms =
- stats->time_between_rendered_frames_ms.GetAverage();
- if (time_between_rendered_frames.ms() >
- std::max(kFreezeThresholdMs + average_time_between_rendered_frames_ms,
- 3 * average_time_between_rendered_frames_ms)) {
- stats->freeze_time_ms.AddSample(StatsSample(
- time_between_rendered_frames.ms(), frame_stats.rendered_time));
- auto freeze_end_it =
- stream_last_freeze_end_time_.find(comparison.stats_key);
- RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end());
- stats->time_between_freezes_ms.AddSample(StatsSample(
- (frame_stats.prev_frame_rendered_time - freeze_end_it->second).ms(),
- frame_stats.rendered_time));
- freeze_end_it->second = frame_stats.rendered_time;
- }
- }
- }
- // Compute stream codec info.
- if (frame_stats.used_encoder.has_value()) {
- if (stats->encoders.empty() || stats->encoders.back().codec_name !=
- frame_stats.used_encoder->codec_name) {
- stats->encoders.push_back(*frame_stats.used_encoder);
- }
- stats->encoders.back().last_frame_id =
- frame_stats.used_encoder->last_frame_id;
- stats->encoders.back().switched_from_at =
- frame_stats.used_encoder->switched_from_at;
- }
-
- if (frame_stats.used_decoder.has_value()) {
- if (stats->decoders.empty() || stats->decoders.back().codec_name !=
- frame_stats.used_decoder->codec_name) {
- stats->decoders.push_back(*frame_stats.used_decoder);
- }
- stats->decoders.back().last_frame_id =
- frame_stats.used_decoder->last_frame_id;
- stats->decoders.back().switched_from_at =
- frame_stats.used_decoder->switched_from_at;
- }
-}
-
void DefaultVideoQualityAnalyzer::ReportResults() {
using ::webrtc::test::ImproveDirection;
- MutexLock lock1(&lock_);
- MutexLock lock2(&comparison_lock_);
- for (auto& item : stream_stats_) {
+ MutexLock lock(&mutex_);
+ for (auto& item : frames_comparator_.stream_stats()) {
ReportResults(GetTestCaseName(StatsKeyToMetricName(ToStatsKey(item.first))),
item.second, stream_frame_counters_.at(item.first));
}
test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(),
"%", false, ImproveDirection::kSmallerIsBetter);
LogFrameCounters("Global", frame_counters_);
- for (auto& item : stream_stats_) {
+ for (auto& item : frames_comparator_.stream_stats()) {
LogFrameCounters(ToStatsKey(item.first).ToString(),
stream_frame_counters_.at(item.first));
LogStreamInternalStats(ToStatsKey(item.first).ToString(), item.second,
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
index d388b1b..57b202e 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
@@ -30,6 +30,7 @@
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/clock.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
#include "test/pc/e2e/analyzer/video/multi_head_queue.h"
@@ -38,33 +39,11 @@
namespace webrtc {
namespace webrtc_pc_e2e {
-struct DefaultVideoQualityAnalyzerOptions {
- // Tells DefaultVideoQualityAnalyzer if heavy metrics like PSNR and SSIM have
- // to be computed or not.
- bool heavy_metrics_computation_enabled = true;
- // If true DefaultVideoQualityAnalyzer will try to adjust frames before
- // computing PSNR and SSIM for them. In some cases picture may be shifted by
- // a few pixels after the encode/decode step. Those difference is invisible
- // for a human eye, but it affects the metrics. So the adjustment is used to
- // get metrics that are closer to how human persepts the video. This feature
- // significantly slows down the comparison, so turn it on only when it is
- // needed.
- bool adjust_cropping_before_comparing_frames = false;
- // Amount of frames that are queued in the DefaultVideoQualityAnalyzer from
- // the point they were captured to the point they were rendered on all
- // receivers per stream.
- size_t max_frames_in_flight_per_stream_count =
- kDefaultMaxFramesInFlightPerStream;
- // If true, the analyzer will expect peers to receive their own video streams.
- bool enable_receive_own_stream = false;
-};
-
class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public:
explicit DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
- DefaultVideoQualityAnalyzerOptions options =
- DefaultVideoQualityAnalyzerOptions());
+ DefaultVideoQualityAnalyzerOptions options = {});
~DefaultVideoQualityAnalyzer() override;
void Start(std::string test_case_name,
@@ -307,21 +286,12 @@
std::map<absl::string_view, size_t> index_;
};
- void AddComparison(InternalStatsKey stats_key,
- absl::optional<VideoFrame> captured,
- absl::optional<VideoFrame> rendered,
- bool dropped,
- FrameStats frame_stats)
- RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_);
- static void ProcessComparisonsThread(void* obj);
- void ProcessComparisons();
- void ProcessComparison(const FrameComparison& comparison);
// Report results for all metrics for all streams.
void ReportResults();
void ReportResults(const std::string& test_case_name,
const StreamStats& stats,
const FrameCounters& frame_counters)
- RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
@@ -333,28 +303,27 @@
std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now();
StatsKey ToStatsKey(const InternalStatsKey& key) const
- RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Returns string representation of stats key for metrics naming. Used for
// backward compatibility by metrics naming for 2 peers cases.
std::string StatsKeyToMetricName(const StatsKey& key) const
- RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- // TODO(titovartem) restore const when old constructor will be removed.
- DefaultVideoQualityAnalyzerOptions options_;
+ const DefaultVideoQualityAnalyzerOptions options_;
webrtc::Clock* const clock_;
std::atomic<uint16_t> next_frame_id_{0};
std::string test_label_;
- mutable Mutex lock_;
- std::unique_ptr<NamesCollection> peers_ RTC_GUARDED_BY(lock_);
- State state_ RTC_GUARDED_BY(lock_) = State::kNew;
- Timestamp start_time_ RTC_GUARDED_BY(lock_) = Timestamp::MinusInfinity();
+ mutable Mutex mutex_;
+ std::unique_ptr<NamesCollection> peers_ RTC_GUARDED_BY(mutex_);
+ State state_ RTC_GUARDED_BY(mutex_) = State::kNew;
+ Timestamp start_time_ RTC_GUARDED_BY(mutex_) = Timestamp::MinusInfinity();
// Mapping from stream label to unique size_t value to use in stats and avoid
// extra string copying.
- NamesCollection streams_ RTC_GUARDED_BY(lock_);
+ NamesCollection streams_ RTC_GUARDED_BY(mutex_);
// Frames that were captured by all streams and still aren't rendered on
- // receviers or deemed dropped. Frame with id X can be removed from this map
+ // receivers or deemed dropped. Frame with id X can be removed from this map
// if:
// 1. The frame with id X was received in OnFrameRendered by all expected
// receivers.
@@ -365,36 +334,27 @@
// oldest frame id in this stream. In such case only the frame content
// will be removed, but the map entry will be preserved.
std::map<uint16_t, FrameInFlight> captured_frames_in_flight_
- RTC_GUARDED_BY(lock_);
+ RTC_GUARDED_BY(mutex_);
// Global frames count for all video streams.
- FrameCounters frame_counters_ RTC_GUARDED_BY(lock_);
+ FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_);
// Frame counters per each stream per each receiver.
std::map<InternalStatsKey, FrameCounters> stream_frame_counters_
- RTC_GUARDED_BY(lock_);
+ RTC_GUARDED_BY(mutex_);
// Map from stream index in `streams_` to its StreamState.
- std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(lock_);
+ std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(mutex_);
// Map from stream index in `streams_` to sender peer index in `peers_`.
- std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(lock_);
+ std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(mutex_);
// Stores history mapping between stream index in `streams_` and frame ids.
// Updated when frame id overlap. It required to properly return stream label
// after 1st frame from simulcast streams was already rendered and last is
// still encoding.
std::map<size_t, std::set<uint16_t>> stream_to_frame_id_history_
- RTC_GUARDED_BY(lock_);
-
- mutable Mutex comparison_lock_;
- std::map<InternalStatsKey, StreamStats> stream_stats_
- RTC_GUARDED_BY(comparison_lock_);
- std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_
- RTC_GUARDED_BY(comparison_lock_);
- std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
- AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(comparison_lock_);
-
- std::vector<rtc::PlatformThread> thread_pool_;
- rtc::Event comparison_available_event_;
+ RTC_GUARDED_BY(mutex_);
+ AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_);
DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_;
+ DefaultVideoQualityAnalyzerFramesComparator frames_comparator_;
};
} // namespace webrtc_pc_e2e
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
new file mode 100644
index 0000000..0f5f3bf
--- /dev/null
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
@@ -0,0 +1,359 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
+
+#include <algorithm>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/scoped_refptr.h"
+#include "api/video/i420_buffer.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/platform_thread.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+
+namespace webrtc {
+namespace {
+
+constexpr int kFreezeThresholdMs = 150;
+constexpr int kMaxActiveComparisons = 10;
+
+SamplesStatsCounter::StatsSample StatsSample(double value,
+ Timestamp sampling_time) {
+ return SamplesStatsCounter::StatsSample{value, sampling_time};
+}
+
+} // namespace
+
+void DefaultVideoQualityAnalyzerFramesComparator::Start(int max_threads_count) {
+ for (int i = 0; i < max_threads_count; i++) {
+ thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable(
+ [this] { ProcessComparisons(); },
+ "DefaultVideoQualityAnalyzerFramesComparator-" + std::to_string(i)));
+ }
+ {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kNew) << "Frames comparator is already started";
+ state_ = State::kActive;
+ }
+ cpu_measurer_.StartMeasuringCpuProcessTime();
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::Stop(
+ const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times) {
+ {
+ MutexLock lock(&mutex_);
+ if (state_ == State::kStopped) {
+ return;
+ }
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+ state_ = State::kStopped;
+ }
+ cpu_measurer_.StopMeasuringCpuProcessTime();
+ comparison_available_event_.Set();
+ thread_pool_.clear();
+
+ {
+ MutexLock lock(&mutex_);
+ // Perform final Metrics update. On this place analyzer is stopped and no
+ // one holds any locks.
+
+ // Time between freezes.
+ // Count time since the last freeze to the end of the call as time
+ // between freezes.
+ for (auto& entry : last_rendered_frame_times) {
+ const InternalStatsKey& stats_key = entry.first;
+ const Timestamp& last_rendered_frame_time = entry.second;
+
+ // If there are no freezes in the call we have to report
+ // time_between_freezes_ms as call duration and in such case
+ // `last_rendered_frame_time` for this stream will be stream start time.
+ // If there is freeze, then we need add time from last rendered frame
+ // to last freeze end as time between freezes.
+ stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(
+ StatsSample(last_rendered_frame_time.ms() -
+ stream_last_freeze_end_time_.at(stats_key).ms(),
+ Now()));
+ }
+ }
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::EnsureStatsForStream(
+ size_t stream_index,
+ size_t sender_peer_index,
+ size_t peers_count,
+ Timestamp captured_time,
+ Timestamp start_time) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+
+ for (size_t i = 0; i < peers_count; ++i) {
+ if (i == sender_peer_index && !options_.enable_receive_own_stream) {
+ continue;
+ }
+ InternalStatsKey stats_key(stream_index, sender_peer_index, i);
+ if (stream_stats_.find(stats_key) == stream_stats_.end()) {
+ stream_stats_.insert(
+ {stats_key, webrtc_pc_e2e::StreamStats(captured_time)});
+ // Assume that the first freeze was before first stream frame captured.
+ // This way time before the first freeze would be counted as time
+ // between freezes.
+ stream_last_freeze_end_time_.insert({stats_key, start_time});
+ } else {
+ // When we see some `stream_label` for the first time we need to create
+ // stream stats object for it and set up some states, but we need to do
+ // it only once and for all receivers, so on the next frame on the same
+ // `stream_label` we can be sure, that it's already done and we needn't
+ // to scan though all peers again.
+ break;
+ }
+ }
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::RegisterParticipantInCall(
+ rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> stream_started_time,
+ Timestamp start_time) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+
+ for (const std::pair<InternalStatsKey, Timestamp>& pair :
+ stream_started_time) {
+ stream_stats_.insert({pair.first, webrtc_pc_e2e::StreamStats(pair.second)});
+ stream_last_freeze_end_time_.insert({pair.first, start_time});
+ }
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
+ InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+ AddComparisonInternal(std::move(stats_key), std::move(captured),
+ std::move(rendered), dropped, std::move(frame_stats));
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
+ InternalStatsKey stats_key,
+ int skipped_between_rendered,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+ stream_stats_.at(stats_key).skipped_between_rendered.AddSample(
+ StatsSample(skipped_between_rendered, Now()));
+ AddComparisonInternal(std::move(stats_key), std::move(captured),
+ std::move(rendered), dropped, std::move(frame_stats));
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal(
+ InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats) {
+ cpu_measurer_.StartExcludingCpuThreadTime();
+ frames_comparator_stats_.comparisons_queue_size.AddSample(
+ StatsSample(comparisons_.size(), Now()));
+ // If there too many computations waiting in the queue, we won't provide
+ // frames itself to make future computations lighter.
+ if (comparisons_.size() >= kMaxActiveComparisons) {
+ comparisons_.emplace_back(std::move(stats_key), absl::nullopt,
+ absl::nullopt, dropped, std::move(frame_stats),
+ OverloadReason::kCpu);
+ } else {
+ OverloadReason overload_reason = OverloadReason::kNone;
+ if (!captured && !dropped) {
+ overload_reason = OverloadReason::kMemory;
+ }
+ comparisons_.emplace_back(std::move(stats_key), std::move(captured),
+ std::move(rendered), dropped,
+ std::move(frame_stats), overload_reason);
+ }
+ comparison_available_event_.Set();
+ cpu_measurer_.StopExcludingCpuThreadTime();
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparisons() {
+ while (true) {
+ // Try to pick next comparison to perform from the queue.
+ absl::optional<FrameComparison> comparison = absl::nullopt;
+ {
+ MutexLock lock(&mutex_);
+ if (!comparisons_.empty()) {
+ comparison = comparisons_.front();
+ comparisons_.pop_front();
+ if (!comparisons_.empty()) {
+ comparison_available_event_.Set();
+ }
+ }
+ }
+ if (!comparison) {
+ bool more_frames_expected;
+ {
+ // If there are no comparisons and state is stopped =>
+ // no more frames expected.
+ MutexLock lock(&mutex_);
+ more_frames_expected = state_ != State::kStopped;
+ }
+ if (!more_frames_expected) {
+ comparison_available_event_.Set();
+ return;
+ }
+ comparison_available_event_.Wait(1000);
+ continue;
+ }
+
+ cpu_measurer_.StartExcludingCpuThreadTime();
+ ProcessComparison(comparison.value());
+ cpu_measurer_.StopExcludingCpuThreadTime();
+ }
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
+ const FrameComparison& comparison) {
+ // Perform expensive psnr and ssim calculations while not holding lock.
+ double psnr = -1.0;
+ double ssim = -1.0;
+ if (options_.heavy_metrics_computation_enabled && comparison.captured &&
+ !comparison.dropped) {
+ rtc::scoped_refptr<I420BufferInterface> reference_buffer =
+ comparison.captured->video_frame_buffer()->ToI420();
+ rtc::scoped_refptr<I420BufferInterface> test_buffer =
+ comparison.rendered->video_frame_buffer()->ToI420();
+ if (options_.adjust_cropping_before_comparing_frames) {
+ test_buffer =
+ ScaleVideoFrameBuffer(*test_buffer.get(), reference_buffer->width(),
+ reference_buffer->height());
+ reference_buffer = test::AdjustCropping(reference_buffer, test_buffer);
+ }
+ psnr = I420PSNR(*reference_buffer.get(), *test_buffer.get());
+ ssim = I420SSIM(*reference_buffer.get(), *test_buffer.get());
+ }
+
+ const FrameStats& frame_stats = comparison.frame_stats;
+
+ MutexLock lock(&mutex_);
+ auto stats_it = stream_stats_.find(comparison.stats_key);
+ RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString();
+ webrtc_pc_e2e::StreamStats* stats = &stats_it->second;
+ frames_comparator_stats_.comparisons_done++;
+ if (comparison.overload_reason == OverloadReason::kCpu) {
+ frames_comparator_stats_.cpu_overloaded_comparisons_done++;
+ } else if (comparison.overload_reason == OverloadReason::kMemory) {
+ frames_comparator_stats_.memory_overloaded_comparisons_done++;
+ }
+ if (psnr > 0) {
+ stats->psnr.AddSample(StatsSample(psnr, frame_stats.rendered_time));
+ }
+ if (ssim > 0) {
+ stats->ssim.AddSample(StatsSample(ssim, frame_stats.received_time));
+ }
+ if (frame_stats.encoded_time.IsFinite()) {
+ stats->encode_time_ms.AddSample(StatsSample(
+ (frame_stats.encoded_time - frame_stats.pre_encode_time).ms(),
+ frame_stats.encoded_time));
+ stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
+ stats->total_encoded_images_payload += frame_stats.encoded_image_size;
+ stats->target_encode_bitrate.AddSample(StatsSample(
+ frame_stats.target_encode_bitrate, frame_stats.encoded_time));
+ } else {
+ if (frame_stats.pre_encode_time.IsFinite()) {
+ stats->dropped_by_encoder++;
+ } else {
+ stats->dropped_before_encoder++;
+ }
+ }
+ // Next stats can be calculated only if frame was received on remote side.
+ if (!comparison.dropped) {
+ stats->resolution_of_rendered_frame.AddSample(
+ StatsSample(*comparison.frame_stats.rendered_frame_width *
+ *comparison.frame_stats.rendered_frame_height,
+ frame_stats.rendered_time));
+ stats->transport_time_ms.AddSample(StatsSample(
+ (frame_stats.decode_start_time - frame_stats.encoded_time).ms(),
+ frame_stats.received_time));
+ stats->total_delay_incl_transport_ms.AddSample(StatsSample(
+ (frame_stats.rendered_time - frame_stats.captured_time).ms(),
+ frame_stats.received_time));
+ stats->decode_time_ms.AddSample(StatsSample(
+ (frame_stats.decode_end_time - frame_stats.decode_start_time).ms(),
+ frame_stats.decode_end_time));
+ stats->receive_to_render_time_ms.AddSample(StatsSample(
+ (frame_stats.rendered_time - frame_stats.received_time).ms(),
+ frame_stats.rendered_time));
+
+ if (frame_stats.prev_frame_rendered_time.IsFinite()) {
+ TimeDelta time_between_rendered_frames =
+ frame_stats.rendered_time - frame_stats.prev_frame_rendered_time;
+ stats->time_between_rendered_frames_ms.AddSample(StatsSample(
+ time_between_rendered_frames.ms(), frame_stats.rendered_time));
+ double average_time_between_rendered_frames_ms =
+ stats->time_between_rendered_frames_ms.GetAverage();
+ if (time_between_rendered_frames.ms() >
+ std::max(kFreezeThresholdMs + average_time_between_rendered_frames_ms,
+ 3 * average_time_between_rendered_frames_ms)) {
+ stats->freeze_time_ms.AddSample(StatsSample(
+ time_between_rendered_frames.ms(), frame_stats.rendered_time));
+ auto freeze_end_it =
+ stream_last_freeze_end_time_.find(comparison.stats_key);
+ RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end());
+ stats->time_between_freezes_ms.AddSample(StatsSample(
+ (frame_stats.prev_frame_rendered_time - freeze_end_it->second).ms(),
+ frame_stats.rendered_time));
+ freeze_end_it->second = frame_stats.rendered_time;
+ }
+ }
+ }
+ // Compute stream codec info.
+ if (frame_stats.used_encoder.has_value()) {
+ if (stats->encoders.empty() || stats->encoders.back().codec_name !=
+ frame_stats.used_encoder->codec_name) {
+ stats->encoders.push_back(*frame_stats.used_encoder);
+ }
+ stats->encoders.back().last_frame_id =
+ frame_stats.used_encoder->last_frame_id;
+ stats->encoders.back().switched_from_at =
+ frame_stats.used_encoder->switched_from_at;
+ }
+
+ if (frame_stats.used_decoder.has_value()) {
+ if (stats->decoders.empty() || stats->decoders.back().codec_name !=
+ frame_stats.used_decoder->codec_name) {
+ stats->decoders.push_back(*frame_stats.used_decoder);
+ }
+ stats->decoders.back().last_frame_id =
+ frame_stats.used_decoder->last_frame_id;
+ stats->decoders.back().switched_from_at =
+ frame_stats.used_decoder->switched_from_at;
+ }
+}
+
+Timestamp DefaultVideoQualityAnalyzerFramesComparator::Now() {
+ return clock_->CurrentTime();
+}
+
+} // namespace webrtc
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h
new file mode 100644
index 0000000..4a2ac9d
--- /dev/null
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h
@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
+
+#include <deque>
+#include <map>
+#include <utility>
+#include <vector>
+
+#include "api/array_view.h"
+#include "rtc_base/event.h"
+#include "rtc_base/platform_thread.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "system_wrappers/include/clock.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+
+namespace webrtc {
+
+struct FramesComparatorStats {
+ // Size of analyzer internal comparisons queue, measured when new element
+ // id added to the queue.
+ SamplesStatsCounter comparisons_queue_size;
+ // Number of performed comparisons of 2 video frames from captured and
+ // rendered streams.
+ int64_t comparisons_done = 0;
+ // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is
+ // queued when there are too many not processed comparisons in the queue.
+ // Overloaded comparison doesn't include metrics like SSIM and PSNR that
+ // require heavy computations.
+ int64_t cpu_overloaded_comparisons_done = 0;
+ // Number of memory overloaded comparisons. Comparison is memory overloaded if
+ // it is queued when its captured frame was already removed due to high memory
+ // usage for that video stream.
+ int64_t memory_overloaded_comparisons_done = 0;
+};
+
+// Performs comparisons of added frames and tracks frames related statistics.
+// This class is thread safe.
+class DefaultVideoQualityAnalyzerFramesComparator {
+ public:
+ // Creates frames comparator.
+ // Frames comparator doesn't use `options.enable_receive_own_stream` for any
+ // purposes, because it's unrelated to its functionality.
+ DefaultVideoQualityAnalyzerFramesComparator(
+ webrtc::Clock* clock,
+ DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer,
+ webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options = {})
+ : options_(options), clock_(clock), cpu_measurer_(cpu_measurer) {}
+ ~DefaultVideoQualityAnalyzerFramesComparator() { Stop({}); }
+
+ // Starts frames comparator. This method must be invoked before calling
+ // any other method on this object.
+ void Start(int max_threads_count);
+ // Stops frames comparator. This method will block until all added frame
+ // comparisons will be processed. After `Stop()` is invoked no more new
+ // comparisons can be added to this frames comparator.
+ //
+ // `last_rendered_frame_time` contains timestamps of last rendered frame for
+ // each (stream, sender, receiver) tuple to properly update time between
+ // freezes: it has include time from the last freeze until and of call.
+ void Stop(
+ const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times);
+
+ // Ensures that stream `stream_index` has stats objects created for all
+ // potential receivers. This method must be called before adding any
+ // frames comparison for that stream.
+ void EnsureStatsForStream(size_t stream_index,
+ size_t sender_peer_index,
+ size_t peers_count,
+ Timestamp captured_time,
+ Timestamp start_time);
+ // Ensures that newly added participant will have stream stats objects created
+ // for all streams which they can receive. This method must be called before
+ // any frames comparison will be added for the newly added participant.
+ //
+ // `stream_started_time` - start time of each stream for which stats object
+ // has to be created.
+ // `start_time` - call start time.
+ void RegisterParticipantInCall(
+ rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>>
+ stream_started_time,
+ Timestamp start_time);
+
+ void AddComparison(InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats);
+ // `skipped_between_rendered` - amount of frames dropped on this stream before
+ // last received frame and current frame.
+ void AddComparison(InternalStatsKey stats_key,
+ int skipped_between_rendered,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats);
+
+ std::map<InternalStatsKey, webrtc_pc_e2e::StreamStats> stream_stats() const {
+ MutexLock lock(&mutex_);
+ return stream_stats_;
+ }
+ FramesComparatorStats frames_comparator_stats() const {
+ MutexLock lock(&mutex_);
+ return frames_comparator_stats_;
+ }
+
+ private:
+ enum State { kNew, kActive, kStopped };
+
+ void AddComparisonInternal(InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ bool dropped,
+ FrameStats frame_stats)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ void ProcessComparisons();
+ void ProcessComparison(const FrameComparison& comparison);
+ Timestamp Now();
+
+ const webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options_;
+ webrtc::Clock* const clock_;
+ DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer_;
+
+ mutable Mutex mutex_;
+ State state_ RTC_GUARDED_BY(mutex_) = State::kNew;
+ std::map<InternalStatsKey, webrtc_pc_e2e::StreamStats> stream_stats_
+ RTC_GUARDED_BY(mutex_);
+ std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_
+ RTC_GUARDED_BY(mutex_);
+ std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(mutex_);
+ FramesComparatorStats frames_comparator_stats_ RTC_GUARDED_BY(mutex_);
+
+ std::vector<rtc::PlatformThread> thread_pool_;
+ rtc::Event comparison_available_event_;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc
new file mode 100644
index 0000000..9a63cd0
--- /dev/null
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc
@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
+
+#include <map>
+
+#include "api/test/create_frame_generator.h"
+#include "api/units/timestamp.h"
+#include "system_wrappers/include/clock.h"
+#include "system_wrappers/include/sleep.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+
+namespace webrtc {
+namespace {
+
+using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample;
+
+constexpr int kMaxFramesInFlightPerStream = 10;
+
+webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
+ webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options;
+ options.heavy_metrics_computation_enabled = false;
+ options.adjust_cropping_before_comparing_frames = false;
+ options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
+ return options;
+}
+
+webrtc_pc_e2e::StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id,
+ Timestamp time) {
+ webrtc_pc_e2e::StreamCodecInfo info;
+ info.codec_name = "VP8";
+ info.first_frame_id = frame_id;
+ info.last_frame_id = frame_id;
+ info.switched_on_at = time;
+ info.switched_from_at = time;
+ return info;
+}
+
+FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
+ Timestamp captured_time) {
+ FrameStats frame_stats(captured_time);
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+ frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
+ frame_stats.rendered_time = captured_time + TimeDelta::Millis(60);
+ frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(1, frame_stats.decode_end_time);
+ frame_stats.rendered_frame_width = 10;
+ frame_stats.rendered_frame_height = 10;
+ return frame_stats;
+}
+
+double GetFirstOrDie(const SamplesStatsCounter& counter) {
+ EXPECT_TRUE(!counter.IsEmpty()) << "Counter has to be not empty";
+ return counter.GetSamples()[0];
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ StatsPresentedAfterAddingOneComparison) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
+
+ Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ size_t peers_count = 2;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ FrameStats frame_stats =
+ FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time);
+
+ comparator.Start(1);
+ comparator.EnsureStatsForStream(stream, sender, peers_count,
+ stream_start_time, stream_start_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt, /*dropped=*/false,
+ frame_stats);
+ comparator.Stop({});
+
+ std::map<InternalStatsKey, webrtc_pc_e2e::StreamStats> stats =
+ comparator.stream_stats();
+ EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).transport_time_ms), 20.0);
+ EXPECT_DOUBLE_EQ(
+ GetFirstOrDie(stats.at(stats_key).total_delay_incl_transport_ms), 60.0);
+ EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).encode_time_ms), 10.0);
+ EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).decode_time_ms), 10.0);
+ EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).receive_to_render_time_ms),
+ 30.0);
+ EXPECT_DOUBLE_EQ(
+ GetFirstOrDie(stats.at(stats_key).resolution_of_rendered_frame), 100.0);
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ MultiFrameStatsPresentedAfterAddingTwoComparisonWith10msDelay) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
+
+ Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ size_t peers_count = 2;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ FrameStats frame_stats1 =
+ FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time);
+ FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
+ stream_start_time + TimeDelta::Millis(15));
+ frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time;
+
+ comparator.Start(1);
+ comparator.EnsureStatsForStream(stream, sender, peers_count,
+ stream_start_time, stream_start_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt, /*dropped=*/false,
+ frame_stats1);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt, /*dropped=*/false,
+ frame_stats2);
+ comparator.Stop({});
+
+ std::map<InternalStatsKey, webrtc_pc_e2e::StreamStats> stats =
+ comparator.stream_stats();
+ EXPECT_DOUBLE_EQ(
+ GetFirstOrDie(stats.at(stats_key).time_between_rendered_frames_ms), 15.0);
+ EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(),
+ 2.0 / 15 * 1000);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc
index d823091..1149ac1 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc
@@ -26,7 +26,7 @@
event_first_time_ = event_time;
}
event_last_time_ = event_time;
- event_count_++;
+ events_count_++;
}
double RateCounter::GetEventsPerSecond() const {
@@ -34,7 +34,7 @@
// Divide on us and multiply on kMicrosPerSecond to correctly process cases
// where there were too small amount of events, so difference is less then 1
// sec. We can use us here, because Timestamp has us resolution.
- return static_cast<double>(event_count_) /
+ return static_cast<double>(events_count_) /
(event_last_time_ - event_first_time_).us() * kMicrosPerSecond;
}
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
index b5fdedd..4c90961 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
@@ -39,7 +39,7 @@
private:
Timestamp event_first_time_ = Timestamp::MinusInfinity();
Timestamp event_last_time_ = Timestamp::MinusInfinity();
- int64_t event_count_ = 0;
+ int64_t events_count_ = 0;
};
struct FrameCounters {
@@ -165,6 +165,27 @@
bool operator<(const StatsKey& a, const StatsKey& b);
bool operator==(const StatsKey& a, const StatsKey& b);
+struct DefaultVideoQualityAnalyzerOptions {
+ // Tells DefaultVideoQualityAnalyzer if heavy metrics like PSNR and SSIM have
+ // to be computed or not.
+ bool heavy_metrics_computation_enabled = true;
+ // If true DefaultVideoQualityAnalyzer will try to adjust frames before
+ // computing PSNR and SSIM for them. In some cases picture may be shifted by
+ // a few pixels after the encode/decode step. Those difference is invisible
+ // for a human eye, but it affects the metrics. So the adjustment is used to
+ // get metrics that are closer to how human perceive the video. This feature
+ // significantly slows down the comparison, so turn it on only when it is
+ // needed.
+ bool adjust_cropping_before_comparing_frames = false;
+ // Amount of frames that are queued in the DefaultVideoQualityAnalyzer from
+ // the point they were captured to the point they were rendered on all
+ // receivers per stream.
+ size_t max_frames_in_flight_per_stream_count =
+ kDefaultMaxFramesInFlightPerStream;
+ // If true, the analyzer will expect peers to receive their own video streams.
+ bool enable_receive_own_stream = false;
+};
+
} // namespace webrtc_pc_e2e
} // namespace webrtc