| /* |
| * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" |
| |
| #include <algorithm> |
| #include <map> |
| #include <memory> |
| #include <set> |
| #include <string> |
| #include <utility> |
| #include <vector> |
| |
| #include "api/array_view.h" |
| #include "api/numerics/samples_stats_counter.h" |
| #include "api/test/metrics/metric.h" |
| #include "api/units/time_delta.h" |
| #include "api/units/timestamp.h" |
| #include "api/video/video_frame.h" |
| #include "rtc_base/checks.h" |
| #include "rtc_base/logging.h" |
| #include "rtc_base/strings/string_builder.h" |
| #include "system_wrappers/include/clock.h" |
| #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" |
| #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" |
| #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" |
| #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" |
| #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" |
| #include "test/pc/e2e/metric_metadata_keys.h" |
| |
| namespace webrtc { |
| namespace { |
| |
| using ::webrtc::test::ImprovementDirection; |
| using ::webrtc::test::Unit; |
| using ::webrtc::webrtc_pc_e2e::MetricMetadataKey; |
| |
| constexpr int kBitsInByte = 8; |
| constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed"; |
| constexpr absl::string_view kSkipRenderedFrameReasonRendered = "rendered"; |
| constexpr absl::string_view kSkipRenderedFrameReasonDropped = |
| "considered dropped"; |
| |
| void LogFrameCounters(const std::string& name, const FrameCounters& counters) { |
| RTC_LOG(LS_INFO) << "[" << name |
| << "] Captured : " << counters.captured; |
| RTC_LOG(LS_INFO) << "[" << name |
| << "] Pre encoded : " << counters.pre_encoded; |
| RTC_LOG(LS_INFO) << "[" << name |
| << "] Encoded : " << counters.encoded; |
| RTC_LOG(LS_INFO) << "[" << name |
| << "] Received : " << counters.received; |
| RTC_LOG(LS_INFO) << "[" << name |
| << "] Decoded : " << counters.decoded; |
| RTC_LOG(LS_INFO) << "[" << name |
| << "] Rendered : " << counters.rendered; |
| RTC_LOG(LS_INFO) << "[" << name |
| << "] Dropped : " << counters.dropped; |
| RTC_LOG(LS_INFO) << "[" << name |
| << "] Failed to decode : " << counters.failed_to_decode; |
| } |
| |
| void LogStreamInternalStats(const std::string& name, |
| const StreamStats& stats, |
| Timestamp start_time) { |
| for (const auto& entry : stats.dropped_by_phase) { |
| RTC_LOG(LS_INFO) << "[" << name << "] Dropped at " << ToString(entry.first) |
| << ": " << entry.second; |
| } |
| Timestamp first_encoded_frame_time = Timestamp::PlusInfinity(); |
| for (const StreamCodecInfo& encoder : stats.encoders) { |
| RTC_DCHECK(encoder.switched_on_at.IsFinite()); |
| RTC_DCHECK(encoder.switched_from_at.IsFinite()); |
| if (first_encoded_frame_time.IsInfinite()) { |
| first_encoded_frame_time = encoder.switched_on_at; |
| } |
| RTC_LOG(LS_INFO) |
| << "[" << name << "] Used encoder: \"" << encoder.codec_name |
| << "\" used from (frame_id=" << encoder.first_frame_id |
| << "; from_stream_start=" |
| << (encoder.switched_on_at - stats.stream_started_time).ms() |
| << "ms, from_call_start=" << (encoder.switched_on_at - start_time).ms() |
| << "ms) until (frame_id=" << encoder.last_frame_id |
| << "; from_stream_start=" |
| << (encoder.switched_from_at - stats.stream_started_time).ms() |
| << "ms, from_call_start=" |
| << (encoder.switched_from_at - start_time).ms() << "ms)"; |
| } |
| for (const StreamCodecInfo& decoder : stats.decoders) { |
| RTC_DCHECK(decoder.switched_on_at.IsFinite()); |
| RTC_DCHECK(decoder.switched_from_at.IsFinite()); |
| RTC_LOG(LS_INFO) |
| << "[" << name << "] Used decoder: \"" << decoder.codec_name |
| << "\" used from (frame_id=" << decoder.first_frame_id |
| << "; from_stream_start=" |
| << (decoder.switched_on_at - stats.stream_started_time).ms() |
| << "ms, from_call_start=" << (decoder.switched_on_at - start_time).ms() |
| << "ms) until (frame_id=" << decoder.last_frame_id |
| << "; from_stream_start=" |
| << (decoder.switched_from_at - stats.stream_started_time).ms() |
| << "ms, from_call_start=" |
| << (decoder.switched_from_at - start_time).ms() << "ms)"; |
| } |
| } |
| |
| template <typename T> |
| absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { |
| auto it = map.find(key); |
| if (it == map.end()) { |
| return absl::nullopt; |
| } |
| return it->second; |
| } |
| |
| SamplesStatsCounter::StatsSample StatsSample(double value, |
| Timestamp sampling_time) { |
| return SamplesStatsCounter::StatsSample{value, sampling_time}; |
| } |
| |
| } // namespace |
| |
| DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( |
| webrtc::Clock* clock, |
| test::MetricsLogger* metrics_logger, |
| DefaultVideoQualityAnalyzerOptions options) |
| : options_(options), |
| clock_(clock), |
| metrics_logger_(metrics_logger), |
| frames_storage_(options.max_frames_storage_duration, clock_), |
| frames_comparator_(clock, cpu_measurer_, options) { |
| RTC_CHECK(metrics_logger_); |
| } |
| |
| DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { |
| Stop(); |
| } |
| |
| void DefaultVideoQualityAnalyzer::Start( |
| std::string test_case_name, |
| rtc::ArrayView<const std::string> peer_names, |
| int max_threads_count) { |
| test_label_ = std::move(test_case_name); |
| frames_comparator_.Start(max_threads_count); |
| { |
| MutexLock lock(&mutex_); |
| peers_ = std::make_unique<NamesCollection>(peer_names); |
| RTC_CHECK(start_time_.IsMinusInfinity()); |
| |
| RTC_CHECK_EQ(state_, State::kNew) |
| << "DefaultVideoQualityAnalyzer is already started"; |
| state_ = State::kActive; |
| start_time_ = Now(); |
| } |
| } |
| |
| uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( |
| absl::string_view peer_name, |
| const std::string& stream_label, |
| const webrtc::VideoFrame& frame) { |
| // `next_frame_id` is atomic, so we needn't lock here. |
| Timestamp captured_time = Now(); |
| Timestamp start_time = Timestamp::MinusInfinity(); |
| size_t peer_index = -1; |
| size_t peers_count = -1; |
| size_t stream_index; |
| uint16_t frame_id = VideoFrame::kNotSetId; |
| { |
| MutexLock lock(&mutex_); |
| frame_id = GetNextFrameId(); |
| RTC_CHECK_EQ(state_, State::kActive) |
| << "DefaultVideoQualityAnalyzer has to be started before use"; |
| // Create a local copy of `start_time_`, peer's index and total peers count |
| // to access it without holding a `mutex_` during access to |
| // `frames_comparator_`. |
| start_time = start_time_; |
| peer_index = peers_->index(peer_name); |
| peers_count = peers_->size(); |
| stream_index = streams_.AddIfAbsent(stream_label); |
| } |
| // Ensure stats for this stream exists. |
| frames_comparator_.EnsureStatsForStream(stream_index, peer_index, peers_count, |
| captured_time, start_time); |
| { |
| MutexLock lock(&mutex_); |
| stream_to_sender_[stream_index] = peer_index; |
| frame_counters_.captured++; |
| for (size_t i : peers_->GetAllIndexes()) { |
| if (i != peer_index || options_.enable_receive_own_stream) { |
| InternalStatsKey key(stream_index, peer_index, i); |
| stream_frame_counters_[key].captured++; |
| } |
| } |
| |
| std::set<size_t> frame_receivers_indexes = peers_->GetPresentIndexes(); |
| if (!options_.enable_receive_own_stream) { |
| frame_receivers_indexes.erase(peer_index); |
| } |
| |
| auto state_it = stream_states_.find(stream_index); |
| if (state_it == stream_states_.end()) { |
| stream_states_.emplace(stream_index, |
| StreamState(peer_index, frame_receivers_indexes, |
| captured_time, clock_)); |
| } |
| StreamState* state = &stream_states_.at(stream_index); |
| state->PushBack(frame_id); |
| absl::optional<TimeDelta> time_between_captured_frames = absl::nullopt; |
| if (state->last_captured_frame_time().has_value()) { |
| time_between_captured_frames = |
| captured_time - *state->last_captured_frame_time(); |
| } |
| state->SetLastCapturedFrameTime(captured_time); |
| // Update frames in flight info. |
| auto it = captured_frames_in_flight_.find(frame_id); |
| if (it != captured_frames_in_flight_.end()) { |
| // If we overflow uint16_t and hit previous frame id and this frame is |
| // still in flight, it means that this stream wasn't rendered for long |
| // time and we need to process existing frame as dropped. |
| for (size_t i : peers_->GetPresentIndexes()) { |
| if (i == peer_index && !options_.enable_receive_own_stream) { |
| continue; |
| } |
| |
| uint16_t oldest_frame_id = state->PopFront(i); |
| RTC_DCHECK_EQ(frame_id, oldest_frame_id); |
| |
| if (state->GetPausableState(i)->IsPaused()) { |
| continue; |
| } |
| |
| frame_counters_.dropped++; |
| InternalStatsKey key(stream_index, peer_index, i); |
| stream_frame_counters_.at(key).dropped++; |
| |
| analyzer_stats_.frames_in_flight_left_count.AddSample( |
| StatsSample(captured_frames_in_flight_.size(), Now())); |
| frames_comparator_.AddComparison( |
| InternalStatsKey(stream_index, peer_index, i), |
| /*captured=*/absl::nullopt, |
| /*rendered=*/absl::nullopt, FrameComparisonType::kDroppedFrame, |
| it->second.GetStatsForPeer(i)); |
| } |
| |
| frames_storage_.Remove(it->second.id()); |
| captured_frames_in_flight_.erase(it); |
| } |
| captured_frames_in_flight_.emplace( |
| frame_id, FrameInFlight(stream_index, frame_id, captured_time, |
| time_between_captured_frames, |
| std::move(frame_receivers_indexes))); |
| // Store local copy of the frame with frame_id set. |
| VideoFrame local_frame(frame); |
| local_frame.set_id(frame_id); |
| frames_storage_.Add(std::move(local_frame), captured_time); |
| |
| // Update history stream<->frame mapping |
| for (auto it = stream_to_frame_id_history_.begin(); |
| it != stream_to_frame_id_history_.end(); ++it) { |
| it->second.erase(frame_id); |
| } |
| stream_to_frame_id_history_[stream_index].insert(frame_id); |
| stream_to_frame_id_full_history_[stream_index].push_back(frame_id); |
| |
| if (options_.report_infra_metrics) { |
| analyzer_stats_.on_frame_captured_processing_time_ms.AddSample( |
| (Now() - captured_time).ms<double>()); |
| } |
| } |
| return frame_id; |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnFramePreEncode( |
| absl::string_view peer_name, |
| const webrtc::VideoFrame& frame) { |
| Timestamp processing_started = Now(); |
| MutexLock lock(&mutex_); |
| RTC_CHECK_EQ(state_, State::kActive) |
| << "DefaultVideoQualityAnalyzer has to be started before use"; |
| |
| auto it = captured_frames_in_flight_.find(frame.id()); |
| RTC_CHECK(it != captured_frames_in_flight_.end()) |
| << "Frame id=" << frame.id() << " not found"; |
| FrameInFlight& frame_in_flight = it->second; |
| frame_counters_.pre_encoded++; |
| size_t peer_index = peers_->index(peer_name); |
| for (size_t i : peers_->GetAllIndexes()) { |
| if (i != peer_index || options_.enable_receive_own_stream) { |
| InternalStatsKey key(frame_in_flight.stream(), peer_index, i); |
| stream_frame_counters_.at(key).pre_encoded++; |
| } |
| } |
| frame_in_flight.SetPreEncodeTime(Now()); |
| |
| if (options_.report_infra_metrics) { |
| analyzer_stats_.on_frame_pre_encode_processing_time_ms.AddSample( |
| (Now() - processing_started).ms<double>()); |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnFrameEncoded( |
| absl::string_view peer_name, |
| uint16_t frame_id, |
| const webrtc::EncodedImage& encoded_image, |
| const EncoderStats& stats, |
| bool discarded) { |
| if (discarded) |
| return; |
| |
| Timestamp processing_started = Now(); |
| MutexLock lock(&mutex_); |
| RTC_CHECK_EQ(state_, State::kActive) |
| << "DefaultVideoQualityAnalyzer has to be started before use"; |
| |
| auto it = captured_frames_in_flight_.find(frame_id); |
| if (it == captured_frames_in_flight_.end()) { |
| RTC_LOG(LS_WARNING) |
| << "The encoding of video frame with id [" << frame_id << "] for peer [" |
| << peer_name << "] finished after all receivers rendered this frame or " |
| << "were removed. It can be OK for simulcast/SVC if higher quality " |
| << "stream is not required or the last receiver was unregistered " |
| << "between encoding of different layers, but it may indicate an ERROR " |
| << "for singlecast or if it happens often."; |
| return; |
| } |
| FrameInFlight& frame_in_flight = it->second; |
| // For SVC we can receive multiple encoded images for one frame, so to cover |
| // all cases we have to pick the last encode time. |
| if (!frame_in_flight.HasEncodedTime()) { |
| // Increase counters only when we meet this frame first time. |
| frame_counters_.encoded++; |
| size_t peer_index = peers_->index(peer_name); |
| for (size_t i : peers_->GetAllIndexes()) { |
| if (i != peer_index || options_.enable_receive_own_stream) { |
| InternalStatsKey key(frame_in_flight.stream(), peer_index, i); |
| stream_frame_counters_.at(key).encoded++; |
| } |
| } |
| } |
| Timestamp now = Now(); |
| StreamState& state = stream_states_.at(frame_in_flight.stream()); |
| absl::optional<TimeDelta> time_between_encoded_frames = absl::nullopt; |
| if (state.last_encoded_frame_time().has_value()) { |
| time_between_encoded_frames = now - *state.last_encoded_frame_time(); |
| } |
| state.SetLastEncodedFrameTime(now); |
| |
| StreamCodecInfo used_encoder; |
| used_encoder.codec_name = stats.encoder_name; |
| used_encoder.first_frame_id = frame_id; |
| used_encoder.last_frame_id = frame_id; |
| used_encoder.switched_on_at = now; |
| used_encoder.switched_from_at = now; |
| // We could either have simulcast layers or spatial layers. |
| // TODO(https://crbug.com/webrtc/14891): If we want to support a mix of |
| // simulcast and SVC we'll also need to consider the case where we have both |
| // simulcast and spatial indices. |
| size_t stream_index = encoded_image.SpatialIndex().value_or( |
| encoded_image.SimulcastIndex().value_or(0)); |
| frame_in_flight.OnFrameEncoded( |
| now, time_between_encoded_frames, encoded_image._frameType, |
| DataSize::Bytes(encoded_image.size()), stats.target_encode_bitrate, |
| stream_index, stats.qp, used_encoder); |
| |
| if (options_.report_infra_metrics) { |
| analyzer_stats_.on_frame_encoded_processing_time_ms.AddSample( |
| (Now() - processing_started).ms<double>()); |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnFrameDropped( |
| absl::string_view peer_name, |
| webrtc::EncodedImageCallback::DropReason reason) { |
| // Here we do nothing, because we will see this drop on renderer side. |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnFramePreDecode( |
| absl::string_view peer_name, |
| uint16_t frame_id, |
| const webrtc::EncodedImage& input_image) { |
| Timestamp processing_started = Now(); |
| MutexLock lock(&mutex_); |
| RTC_CHECK_EQ(state_, State::kActive) |
| << "DefaultVideoQualityAnalyzer has to be started before use"; |
| |
| size_t peer_index = peers_->index(peer_name); |
| |
| if (frame_id == VideoFrame::kNotSetId) { |
| frame_counters_.received++; |
| unknown_sender_frame_counters_[std::string(peer_name)].received++; |
| return; |
| } |
| |
| auto it = captured_frames_in_flight_.find(frame_id); |
| if (it == captured_frames_in_flight_.end() || |
| it->second.HasReceivedTime(peer_index)) { |
| // It means this frame was predecoded before, so we can skip it. It may |
| // happen when we have multiple simulcast streams in one track and received |
| // the same picture from two different streams because SFU can't reliably |
| // correlate two simulcast streams and started relaying the second stream |
| // from the same frame it has relayed right before for the first stream. |
| return; |
| } |
| |
| frame_counters_.received++; |
| InternalStatsKey key(it->second.stream(), |
| stream_to_sender_.at(it->second.stream()), peer_index); |
| stream_frame_counters_.at(key).received++; |
| // Determine the time of the last received packet of this video frame. |
| RTC_DCHECK(!input_image.PacketInfos().empty()); |
| Timestamp last_receive_time = |
| std::max_element(input_image.PacketInfos().cbegin(), |
| input_image.PacketInfos().cend(), |
| [](const RtpPacketInfo& a, const RtpPacketInfo& b) { |
| return a.receive_time() < b.receive_time(); |
| }) |
| ->receive_time(); |
| it->second.OnFramePreDecode(peer_index, |
| /*received_time=*/last_receive_time, |
| /*decode_start_time=*/Now(), |
| input_image._frameType, |
| DataSize::Bytes(input_image.size())); |
| |
| if (options_.report_infra_metrics) { |
| analyzer_stats_.on_frame_pre_decode_processing_time_ms.AddSample( |
| (Now() - processing_started).ms<double>()); |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnFrameDecoded( |
| absl::string_view peer_name, |
| const webrtc::VideoFrame& frame, |
| const DecoderStats& stats) { |
| Timestamp processing_started = Now(); |
| MutexLock lock(&mutex_); |
| RTC_CHECK_EQ(state_, State::kActive) |
| << "DefaultVideoQualityAnalyzer has to be started before use"; |
| |
| size_t peer_index = peers_->index(peer_name); |
| |
| if (frame.id() == VideoFrame::kNotSetId) { |
| frame_counters_.decoded++; |
| unknown_sender_frame_counters_[std::string(peer_name)].decoded++; |
| return; |
| } |
| |
| auto it = captured_frames_in_flight_.find(frame.id()); |
| if (it == captured_frames_in_flight_.end() || |
| it->second.HasDecodeEndTime(peer_index)) { |
| // It means this frame was decoded before, so we can skip it. It may happen |
| // when we have multiple simulcast streams in one track and received |
| // the same frame from two different streams because SFU can't reliably |
| // correlate two simulcast streams and started relaying the second stream |
| // from the same frame it has relayed right before for the first stream. |
| return; |
| } |
| frame_counters_.decoded++; |
| InternalStatsKey key(it->second.stream(), |
| stream_to_sender_.at(it->second.stream()), peer_index); |
| stream_frame_counters_.at(key).decoded++; |
| Timestamp now = Now(); |
| StreamCodecInfo used_decoder; |
| used_decoder.codec_name = stats.decoder_name; |
| used_decoder.first_frame_id = frame.id(); |
| used_decoder.last_frame_id = frame.id(); |
| used_decoder.switched_on_at = now; |
| used_decoder.switched_from_at = now; |
| it->second.OnFrameDecoded(peer_index, now, frame.width(), frame.height(), |
| used_decoder); |
| |
| if (options_.report_infra_metrics) { |
| analyzer_stats_.on_frame_decoded_processing_time_ms.AddSample( |
| (Now() - processing_started).ms<double>()); |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnFrameRendered( |
| absl::string_view peer_name, |
| const webrtc::VideoFrame& frame) { |
| Timestamp processing_started = Now(); |
| MutexLock lock(&mutex_); |
| RTC_CHECK_EQ(state_, State::kActive) |
| << "DefaultVideoQualityAnalyzer has to be started before use"; |
| |
| size_t peer_index = peers_->index(peer_name); |
| |
| if (frame.id() == VideoFrame::kNotSetId) { |
| frame_counters_.rendered++; |
| unknown_sender_frame_counters_[std::string(peer_name)].rendered++; |
| return; |
| } |
| |
| auto frame_it = captured_frames_in_flight_.find(frame.id()); |
| if (frame_it == captured_frames_in_flight_.end() || |
| frame_it->second.HasRenderedTime(peer_index) || |
| frame_it->second.IsDropped(peer_index)) { |
| // It means this frame was rendered or dropped before, so we can skip it. |
| // It may happen when we have multiple simulcast streams in one track and |
| // received the same frame from two different streams because SFU can't |
| // reliably correlate two simulcast streams and started relaying the second |
| // stream from the same frame it has relayed right before for the first |
| // stream. |
| absl::string_view reason = kSkipRenderedFrameReasonProcessed; |
| if (frame_it != captured_frames_in_flight_.end()) { |
| if (frame_it->second.HasRenderedTime(peer_index)) { |
| reason = kSkipRenderedFrameReasonRendered; |
| } else if (frame_it->second.IsDropped(peer_index)) { |
| reason = kSkipRenderedFrameReasonDropped; |
| } |
| } |
| RTC_LOG(LS_WARNING) |
| << "Peer " << peer_name |
| << "; Received frame out of order: received frame with id " |
| << frame.id() << " which was " << reason << " before"; |
| return; |
| } |
| |
| // Find corresponding captured frame. |
| FrameInFlight* frame_in_flight = &frame_it->second; |
| absl::optional<VideoFrame> captured_frame = frames_storage_.Get(frame.id()); |
| |
| const size_t stream_index = frame_in_flight->stream(); |
| StreamState* state = &stream_states_.at(stream_index); |
| const InternalStatsKey stats_key(stream_index, state->sender(), peer_index); |
| |
| // Update frames counters. |
| frame_counters_.rendered++; |
| stream_frame_counters_.at(stats_key).rendered++; |
| |
| // Update current frame stats. |
| frame_in_flight->OnFrameRendered(peer_index, Now()); |
| |
| // After we received frame here we need to check if there are any dropped |
| // frames between this one and last one, that was rendered for this video |
| // stream. |
| int dropped_count = ProcessNotSeenFramesBeforeRendered(peer_index, frame.id(), |
| stats_key, *state); |
| RTC_DCHECK(!state->IsEmpty(peer_index)); |
| state->PopFront(peer_index); |
| |
| if (state->last_rendered_frame_time(peer_index).has_value()) { |
| TimeDelta time_between_rendered_frames = |
| state->GetPausableState(peer_index) |
| ->GetActiveDurationFrom( |
| *state->last_rendered_frame_time(peer_index)); |
| if (state->GetPausableState(peer_index)->IsPaused()) { |
| // If stream is currently paused for this receiver, but we still received |
| // frame, we have to add time from last pause up to Now() to the time |
| // between rendered frames. |
| time_between_rendered_frames += |
| Now() - state->GetPausableState(peer_index)->GetLastEventTime(); |
| } |
| frame_in_flight->SetTimeBetweenRenderedFrames(peer_index, |
| time_between_rendered_frames); |
| frame_in_flight->SetPrevFrameRenderedTime( |
| peer_index, *state->last_rendered_frame_time(peer_index)); |
| } |
| state->SetLastRenderedFrameTime(peer_index, |
| frame_in_flight->rendered_time(peer_index)); |
| analyzer_stats_.frames_in_flight_left_count.AddSample( |
| StatsSample(captured_frames_in_flight_.size(), Now())); |
| frames_comparator_.AddComparison( |
| stats_key, dropped_count, captured_frame, /*rendered=*/frame, |
| FrameComparisonType::kRegular, |
| frame_in_flight->GetStatsForPeer(peer_index)); |
| |
| if (frame_it->second.HaveAllPeersReceived()) { |
| frames_storage_.Remove(frame_it->second.id()); |
| captured_frames_in_flight_.erase(frame_it); |
| } |
| |
| if (options_.report_infra_metrics) { |
| analyzer_stats_.on_frame_rendered_processing_time_ms.AddSample( |
| (Now() - processing_started).ms<double>()); |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnEncoderError( |
| absl::string_view peer_name, |
| const webrtc::VideoFrame& frame, |
| int32_t error_code) { |
| RTC_LOG(LS_ERROR) << "Encoder error for frame.id=" << frame.id() |
| << ", code=" << error_code; |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name, |
| uint16_t frame_id, |
| int32_t error_code, |
| const DecoderStats& stats) { |
| RTC_LOG(LS_ERROR) << "Decoder error for frame_id=" << frame_id |
| << ", code=" << error_code; |
| |
| Timestamp processing_started = Now(); |
| MutexLock lock(&mutex_); |
| RTC_CHECK_EQ(state_, State::kActive) |
| << "DefaultVideoQualityAnalyzer has to be started before use"; |
| |
| size_t peer_index = peers_->index(peer_name); |
| |
| if (frame_id == VideoFrame::kNotSetId) { |
| frame_counters_.failed_to_decode++; |
| unknown_sender_frame_counters_[std::string(peer_name)].failed_to_decode++; |
| return; |
| } |
| |
| auto it = captured_frames_in_flight_.find(frame_id); |
| if (it == captured_frames_in_flight_.end() || |
| it->second.HasDecodeEndTime(peer_index)) { |
| // It means this frame was decoded before, so we can skip it. It may happen |
| // when we have multiple simulcast streams in one track and received |
| // the same frame from two different streams because SFU can't reliably |
| // correlate two simulcast streams and started relaying the second stream |
| // from the same frame it has relayed right before for the first stream. |
| return; |
| } |
| frame_counters_.failed_to_decode++; |
| InternalStatsKey key(it->second.stream(), |
| stream_to_sender_.at(it->second.stream()), peer_index); |
| stream_frame_counters_.at(key).failed_to_decode++; |
| Timestamp now = Now(); |
| StreamCodecInfo used_decoder; |
| used_decoder.codec_name = stats.decoder_name; |
| used_decoder.first_frame_id = frame_id; |
| used_decoder.last_frame_id = frame_id; |
| used_decoder.switched_on_at = now; |
| used_decoder.switched_from_at = now; |
| it->second.OnDecoderError(peer_index, used_decoder); |
| |
| if (options_.report_infra_metrics) { |
| analyzer_stats_.on_decoder_error_processing_time_ms.AddSample( |
| (Now() - processing_started).ms<double>()); |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( |
| absl::string_view peer_name) { |
| MutexLock lock(&mutex_); |
| RTC_CHECK(!peers_->HasName(peer_name)); |
| size_t new_peer_index = peers_->AddIfAbsent(peer_name); |
| |
| // Ensure stats for receiving (for frames from other peers to this one) |
| // streams exists. Since in flight frames will be sent to the new peer |
| // as well. Sending stats (from this peer to others) will be added by |
| // DefaultVideoQualityAnalyzer::OnFrameCaptured. |
| std::vector<std::pair<InternalStatsKey, Timestamp>> stream_started_time; |
| for (auto [stream_index, sender_peer_index] : stream_to_sender_) { |
| InternalStatsKey key(stream_index, sender_peer_index, new_peer_index); |
| |
| // To initiate `FrameCounters` for the stream we should pick frame |
| // counters with the same stream index and the same sender's peer index |
| // and any receiver's peer index and copy from its sender side |
| // counters. |
| FrameCounters counters; |
| for (size_t i : peers_->GetPresentIndexes()) { |
| InternalStatsKey prototype_key(stream_index, sender_peer_index, i); |
| auto it = stream_frame_counters_.find(prototype_key); |
| if (it != stream_frame_counters_.end()) { |
| counters.captured = it->second.captured; |
| counters.pre_encoded = it->second.pre_encoded; |
| counters.encoded = it->second.encoded; |
| break; |
| } |
| } |
| // It may happen if we had only one peer before this method was invoked, |
| // then `counters` will be empty. In such case empty `counters` are ok. |
| stream_frame_counters_.insert({key, std::move(counters)}); |
| |
| stream_started_time.push_back( |
| {key, stream_states_.at(stream_index).stream_started_time()}); |
| } |
| frames_comparator_.RegisterParticipantInCall(stream_started_time, |
| start_time_); |
| // Ensure, that frames states are handled correctly |
| // (e.g. dropped frames tracking). |
| for (auto& [stream_index, stream_state] : stream_states_) { |
| stream_state.AddPeer(new_peer_index); |
| } |
| // Register new peer for every frame in flight. |
| // It is guaranteed, that no garbage FrameInFlight objects will stay in |
| // memory because of adding new peer. Even if the new peer won't receive the |
| // frame, the frame will be removed by OnFrameRendered after next frame comes |
| // for the new peer. It is important because FrameInFlight is a large object. |
| for (auto& [frame_id, frame_in_flight] : captured_frames_in_flight_) { |
| frame_in_flight.AddExpectedReceiver(new_peer_index); |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::UnregisterParticipantInCall( |
| absl::string_view peer_name) { |
| MutexLock lock(&mutex_); |
| RTC_CHECK(peers_->HasName(peer_name)); |
| absl::optional<size_t> peer_index = peers_->RemoveIfPresent(peer_name); |
| RTC_CHECK(peer_index.has_value()); |
| |
| for (auto& [stream_index, stream_state] : stream_states_) { |
| if (!options_.enable_receive_own_stream && |
| peer_index == stream_state.sender()) { |
| continue; |
| } |
| |
| AddExistingFramesInFlightForStreamToComparator(stream_index, stream_state, |
| *peer_index); |
| |
| stream_state.RemovePeer(*peer_index); |
| } |
| |
| // Remove peer from every frame in flight. If we removed that last expected |
| // receiver for the frame, then we should removed this frame if it was |
| // already encoded. If frame wasn't encoded, it still will be used by sender |
| // side pipeline, so we can't delete it yet. |
| for (auto it = captured_frames_in_flight_.begin(); |
| it != captured_frames_in_flight_.end();) { |
| FrameInFlight& frame_in_flight = it->second; |
| frame_in_flight.RemoveExpectedReceiver(*peer_index); |
| // If frame was fully sent and all receivers received it, then erase it. |
| // It may happen that when we remove FrameInFlight only some Simulcast/SVC |
| // layers were encoded and frame has encoded time, but more layers might be |
| // encoded after removal. In such case it's safe to still remove a frame, |
| // because OnFrameEncoded method will correctly handle the case when there |
| // is no FrameInFlight for the received encoded image. |
| if (frame_in_flight.HasEncodedTime() && |
| frame_in_flight.HaveAllPeersReceived()) { |
| frames_storage_.Remove(frame_in_flight.id()); |
| it = captured_frames_in_flight_.erase(it); |
| } else { |
| it++; |
| } |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnPauseAllStreamsFrom( |
| absl::string_view sender_peer_name, |
| absl::string_view receiver_peer_name) { |
| MutexLock lock(&mutex_); |
| if (peers_->HasName(sender_peer_name) && |
| peers_->HasName(receiver_peer_name)) { |
| size_t sender_peer_index = peers_->index(sender_peer_name); |
| size_t receiver_peer_index = peers_->index(receiver_peer_name); |
| for (auto& [unused, stream_state] : stream_states_) { |
| if (stream_state.sender() == sender_peer_index) { |
| stream_state.GetPausableState(receiver_peer_index)->Pause(); |
| } |
| } |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::OnResumeAllStreamsFrom( |
| absl::string_view sender_peer_name, |
| absl::string_view receiver_peer_name) { |
| MutexLock lock(&mutex_); |
| if (peers_->HasName(sender_peer_name) && |
| peers_->HasName(receiver_peer_name)) { |
| size_t sender_peer_index = peers_->index(sender_peer_name); |
| size_t receiver_peer_index = peers_->index(receiver_peer_name); |
| for (auto& [unused, stream_state] : stream_states_) { |
| if (stream_state.sender() == sender_peer_index) { |
| stream_state.GetPausableState(receiver_peer_index)->Resume(); |
| } |
| } |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::Stop() { |
| std::map<InternalStatsKey, Timestamp> last_rendered_frame_times; |
| { |
| MutexLock lock(&mutex_); |
| if (state_ == State::kStopped) { |
| return; |
| } |
| RTC_CHECK_EQ(state_, State::kActive) |
| << "DefaultVideoQualityAnalyzer has to be started before use"; |
| |
| state_ = State::kStopped; |
| |
| // Add the amount of frames in flight to the analyzer stats before all left |
| // frames in flight will be sent to the `frames_compartor_`. |
| analyzer_stats_.frames_in_flight_left_count.AddSample( |
| StatsSample(captured_frames_in_flight_.size(), Now())); |
| |
| for (auto& state_entry : stream_states_) { |
| const size_t stream_index = state_entry.first; |
| StreamState& stream_state = state_entry.second; |
| |
| // Populate `last_rendered_frame_times` map for all peers that were met in |
| // call, not only for the currently presented ones. |
| for (size_t peer_index : peers_->GetAllIndexes()) { |
| if (peer_index == stream_state.sender() && |
| !options_.enable_receive_own_stream) { |
| continue; |
| } |
| |
| InternalStatsKey stats_key(stream_index, stream_state.sender(), |
| peer_index); |
| |
| // If there are no freezes in the call we have to report |
| // time_between_freezes_ms as call duration and in such case |
| // `stream_last_freeze_end_time` for this stream will be `start_time_`. |
| // If there is freeze, then we need add time from last rendered frame |
| // to last freeze end as time between freezes. |
| if (stream_state.last_rendered_frame_time(peer_index)) { |
| last_rendered_frame_times.emplace( |
| stats_key, |
| stream_state.last_rendered_frame_time(peer_index).value()); |
| } |
| } |
| |
| // Push left frame in flight for analysis for the peers that are still in |
| // the call. |
| for (size_t peer_index : peers_->GetPresentIndexes()) { |
| if (peer_index == stream_state.sender() && |
| !options_.enable_receive_own_stream) { |
| continue; |
| } |
| |
| AddExistingFramesInFlightForStreamToComparator( |
| stream_index, stream_state, peer_index); |
| } |
| } |
| } |
| frames_comparator_.Stop(last_rendered_frame_times); |
| |
| // Perform final Metrics update. On this place analyzer is stopped and no one |
| // holds any locks. |
| { |
| MutexLock lock(&mutex_); |
| FramesComparatorStats frames_comparator_stats = |
| frames_comparator_.frames_comparator_stats(); |
| analyzer_stats_.comparisons_queue_size = |
| std::move(frames_comparator_stats.comparisons_queue_size); |
| analyzer_stats_.comparisons_done = frames_comparator_stats.comparisons_done; |
| analyzer_stats_.cpu_overloaded_comparisons_done = |
| frames_comparator_stats.cpu_overloaded_comparisons_done; |
| analyzer_stats_.memory_overloaded_comparisons_done = |
| frames_comparator_stats.memory_overloaded_comparisons_done; |
| } |
| ReportResults(); |
| } |
| |
| std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { |
| MutexLock lock(&mutex_); |
| return GetStreamLabelInternal(frame_id); |
| } |
| |
| std::string DefaultVideoQualityAnalyzer::GetSenderPeerName( |
| uint16_t frame_id) const { |
| MutexLock lock(&mutex_); |
| std::string stream_label = GetStreamLabelInternal(frame_id); |
| size_t stream_index = streams_.index(stream_label); |
| size_t sender_peer_index = stream_states_.at(stream_index).sender(); |
| return peers_->name(sender_peer_index); |
| } |
| |
| std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { |
| MutexLock lock(&mutex_); |
| std::set<StatsKey> out; |
| for (auto& item : frames_comparator_.stream_stats()) { |
| RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " |
| << ToStatsKey(item.first).ToString(); |
| out.insert(ToStatsKey(item.first)); |
| } |
| return out; |
| } |
| |
| VideoStreamsInfo DefaultVideoQualityAnalyzer::GetKnownStreams() const { |
| MutexLock lock(&mutex_); |
| std::map<std::string, std::string> stream_to_sender; |
| std::map<std::string, std::set<std::string>> sender_to_streams; |
| std::map<std::string, std::set<std::string>> stream_to_receivers; |
| |
| for (auto& item : frames_comparator_.stream_stats()) { |
| const std::string& stream_label = streams_.name(item.first.stream); |
| const std::string& sender = peers_->name(item.first.sender); |
| const std::string& receiver = peers_->name(item.first.receiver); |
| RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " |
| << "stream=" << stream_label << "; sender=" << sender |
| << "; receiver=" << receiver; |
| stream_to_sender.emplace(stream_label, sender); |
| auto streams_it = sender_to_streams.find(sender); |
| if (streams_it != sender_to_streams.end()) { |
| streams_it->second.emplace(stream_label); |
| } else { |
| sender_to_streams.emplace(sender, std::set<std::string>{stream_label}); |
| } |
| auto receivers_it = stream_to_receivers.find(stream_label); |
| if (receivers_it != stream_to_receivers.end()) { |
| receivers_it->second.emplace(receiver); |
| } else { |
| stream_to_receivers.emplace(stream_label, |
| std::set<std::string>{receiver}); |
| } |
| } |
| |
| return VideoStreamsInfo(std::move(stream_to_sender), |
| std::move(sender_to_streams), |
| std::move(stream_to_receivers)); |
| } |
| |
| FrameCounters DefaultVideoQualityAnalyzer::GetGlobalCounters() const { |
| MutexLock lock(&mutex_); |
| return frame_counters_; |
| } |
| |
| std::map<std::string, FrameCounters> |
| DefaultVideoQualityAnalyzer::GetUnknownSenderFrameCounters() const { |
| MutexLock lock(&mutex_); |
| return unknown_sender_frame_counters_; |
| } |
| |
| std::map<StatsKey, FrameCounters> |
| DefaultVideoQualityAnalyzer::GetPerStreamCounters() const { |
| MutexLock lock(&mutex_); |
| std::map<StatsKey, FrameCounters> out; |
| for (auto& item : stream_frame_counters_) { |
| out.emplace(ToStatsKey(item.first), item.second); |
| } |
| return out; |
| } |
| |
| std::map<StatsKey, StreamStats> DefaultVideoQualityAnalyzer::GetStats() const { |
| MutexLock lock1(&mutex_); |
| std::map<StatsKey, StreamStats> out; |
| for (auto& item : frames_comparator_.stream_stats()) { |
| out.emplace(ToStatsKey(item.first), item.second); |
| } |
| return out; |
| } |
| |
| AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const { |
| MutexLock lock(&mutex_); |
| return analyzer_stats_; |
| } |
| |
| uint16_t DefaultVideoQualityAnalyzer::GetNextFrameId() { |
| uint16_t frame_id = next_frame_id_++; |
| if (next_frame_id_ == VideoFrame::kNotSetId) { |
| next_frame_id_ = 1; |
| } |
| return frame_id; |
| } |
| |
| void DefaultVideoQualityAnalyzer:: |
| AddExistingFramesInFlightForStreamToComparator(size_t stream_index, |
| StreamState& stream_state, |
| size_t peer_index) { |
| InternalStatsKey stats_key(stream_index, stream_state.sender(), peer_index); |
| |
| // Add frames in flight for this stream into frames comparator. |
| // Frames in flight were not rendered, so they won't affect stream's |
| // last rendered frame time. |
| while (!stream_state.IsEmpty(peer_index) && |
| !stream_state.GetPausableState(peer_index)->IsPaused()) { |
| uint16_t frame_id = stream_state.PopFront(peer_index); |
| auto it = captured_frames_in_flight_.find(frame_id); |
| RTC_DCHECK(it != captured_frames_in_flight_.end()); |
| FrameInFlight& frame = it->second; |
| |
| frames_comparator_.AddComparison(stats_key, /*captured=*/absl::nullopt, |
| /*rendered=*/absl::nullopt, |
| FrameComparisonType::kFrameInFlight, |
| frame.GetStatsForPeer(peer_index)); |
| } |
| } |
| |
| int DefaultVideoQualityAnalyzer::ProcessNotSeenFramesBeforeRendered( |
| size_t peer_index, |
| uint16_t rendered_frame_id, |
| const InternalStatsKey& stats_key, |
| StreamState& state) { |
| int dropped_count = 0; |
| while (!state.IsEmpty(peer_index) && |
| state.Front(peer_index) != rendered_frame_id) { |
| uint16_t next_frame_id = state.PopFront(peer_index); |
| auto next_frame_it = captured_frames_in_flight_.find(next_frame_id); |
| RTC_DCHECK(next_frame_it != captured_frames_in_flight_.end()); |
| FrameInFlight& next_frame = next_frame_it->second; |
| |
| // Depending if the receiver was subscribed to this stream or not at the |
| // time when frame was captured, the frame should be considered as dropped |
| // or superfluous (see below for explanation). Superfluous frames must be |
| // excluded from stats calculations. |
| // |
| // We should consider next cases: |
| // Legend: |
| // + - frame captured on the stream |
| // p - stream is paused |
| // r - stream is resumed |
| // |
| // last currently |
| // rendered rendered |
| // frame frame |
| // |---------------------- dropped -------------------------| |
| // (1) -[]---+---+---+---+---+---+---+---+---+---+---+---+---+---[]-> time |
| // | | |
| // | | |
| // |-- dropped ---┐ ┌- dropped -┐ ┌- dropped ---| |
| // (2) -[]---+---+---+-|-+---+-|-+---+---+-|-+---+-|-+---+---+---[]-> time |
| // | p r p r | |
| // | | |
| // |-- dropped ---┐ ┌------------ dropped ------------| |
| // (3) -[]---+---+---+-|-+---+-|-+---+---+---+---+---+-|-+---+---[]-> time |
| // p r p |
| // |
| // Cases explanation: |
| // (1) Regular media flow, frame is received after freeze. |
| // (2) Stream was paused and received multiple times. Frame is received |
| // after freeze from last resume. |
| // (3) Stream was paused and received multiple times. Frame is received |
| // after stream was paused because frame was already in the network. |
| // |
| // Based on that if stream wasn't paused when `next_frame_id` was captured, |
| // then `next_frame_id` should be considered as dropped. If stream was NOT |
| // resumed after `next_frame_id` was captured but we still received a |
| // `rendered_frame_id` on this stream, then `next_frame_id` also should |
| // be considered as dropped. In other cases `next_frame_id` should be |
| // considered as superfluous, because receiver wasn't expected to receive |
| // `next_frame_id` at all. |
| |
| bool is_dropped = false; |
| bool is_paused = state.GetPausableState(peer_index) |
| ->WasPausedAt(next_frame.captured_time()); |
| if (!is_paused) { |
| is_dropped = true; |
| } else { |
| bool was_resumed_after = |
| state.GetPausableState(peer_index) |
| ->WasResumedAfter(next_frame.captured_time()); |
| if (!was_resumed_after) { |
| is_dropped = true; |
| } |
| } |
| |
| if (is_dropped) { |
| dropped_count++; |
| // Frame with id `dropped_frame_id` was dropped. We need: |
| // 1. Update global and stream frame counters |
| // 2. Extract corresponding frame from `captured_frames_in_flight_` |
| // 3. Send extracted frame to comparison with dropped=true |
| // 4. Cleanup dropped frame |
| frame_counters_.dropped++; |
| stream_frame_counters_.at(stats_key).dropped++; |
| |
| next_frame.MarkDropped(peer_index); |
| |
| analyzer_stats_.frames_in_flight_left_count.AddSample( |
| StatsSample(captured_frames_in_flight_.size(), Now())); |
| frames_comparator_.AddComparison(stats_key, /*captured=*/absl::nullopt, |
| /*rendered=*/absl::nullopt, |
| FrameComparisonType::kDroppedFrame, |
| next_frame.GetStatsForPeer(peer_index)); |
| } else { |
| next_frame.MarkSuperfluous(peer_index); |
| } |
| |
| if (next_frame_it->second.HaveAllPeersReceived()) { |
| frames_storage_.Remove(next_frame_it->second.id()); |
| captured_frames_in_flight_.erase(next_frame_it); |
| } |
| } |
| return dropped_count; |
| } |
| |
| void DefaultVideoQualityAnalyzer::ReportResults() { |
| MutexLock lock(&mutex_); |
| for (auto& item : frames_comparator_.stream_stats()) { |
| ReportResults(item.first, item.second, |
| stream_frame_counters_.at(item.first)); |
| } |
| // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. |
| metrics_logger_->LogSingleValueMetric( |
| "cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless, |
| ImprovementDirection::kSmallerIsBetter, |
| {{MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}}); |
| LogFrameCounters("Global", frame_counters_); |
| if (!unknown_sender_frame_counters_.empty()) { |
| RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:"; |
| for (const auto& [peer_name, frame_counters] : |
| unknown_sender_frame_counters_) { |
| LogFrameCounters(peer_name, frame_counters); |
| } |
| } |
| RTC_LOG(LS_INFO) << "Received frame counters per stream:"; |
| for (const auto& [stats_key, stream_stats] : |
| frames_comparator_.stream_stats()) { |
| LogFrameCounters(ToStatsKey(stats_key).ToString(), |
| stream_frame_counters_.at(stats_key)); |
| LogStreamInternalStats(ToStatsKey(stats_key).ToString(), stream_stats, |
| start_time_); |
| } |
| if (!analyzer_stats_.comparisons_queue_size.IsEmpty()) { |
| RTC_LOG(LS_INFO) << "comparisons_queue_size min=" |
| << analyzer_stats_.comparisons_queue_size.GetMin() |
| << "; max=" |
| << analyzer_stats_.comparisons_queue_size.GetMax() |
| << "; 99%=" |
| << analyzer_stats_.comparisons_queue_size.GetPercentile( |
| 0.99); |
| } |
| RTC_LOG(LS_INFO) << "comparisons_done=" << analyzer_stats_.comparisons_done; |
| RTC_LOG(LS_INFO) << "cpu_overloaded_comparisons_done=" |
| << analyzer_stats_.cpu_overloaded_comparisons_done; |
| RTC_LOG(LS_INFO) << "memory_overloaded_comparisons_done=" |
| << analyzer_stats_.memory_overloaded_comparisons_done; |
| if (options_.report_infra_metrics) { |
| metrics_logger_->LogMetric("comparisons_queue_size", test_label_, |
| analyzer_stats_.comparisons_queue_size, |
| Unit::kCount, |
| ImprovementDirection::kSmallerIsBetter); |
| metrics_logger_->LogMetric("frames_in_flight_left_count", test_label_, |
| analyzer_stats_.frames_in_flight_left_count, |
| Unit::kCount, |
| ImprovementDirection::kSmallerIsBetter); |
| metrics_logger_->LogSingleValueMetric( |
| "comparisons_done", test_label_, analyzer_stats_.comparisons_done, |
| Unit::kCount, ImprovementDirection::kNeitherIsBetter); |
| metrics_logger_->LogSingleValueMetric( |
| "cpu_overloaded_comparisons_done", test_label_, |
| analyzer_stats_.cpu_overloaded_comparisons_done, Unit::kCount, |
| ImprovementDirection::kNeitherIsBetter); |
| metrics_logger_->LogSingleValueMetric( |
| "memory_overloaded_comparisons_done", test_label_, |
| analyzer_stats_.memory_overloaded_comparisons_done, Unit::kCount, |
| ImprovementDirection::kNeitherIsBetter); |
| metrics_logger_->LogSingleValueMetric( |
| "test_duration", test_label_, (Now() - start_time_).ms(), |
| Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter); |
| |
| metrics_logger_->LogMetric( |
| "on_frame_captured_processing_time_ms", test_label_, |
| analyzer_stats_.on_frame_captured_processing_time_ms, |
| Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); |
| metrics_logger_->LogMetric( |
| "on_frame_pre_encode_processing_time_ms", test_label_, |
| analyzer_stats_.on_frame_pre_encode_processing_time_ms, |
| Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); |
| metrics_logger_->LogMetric( |
| "on_frame_encoded_processing_time_ms", test_label_, |
| analyzer_stats_.on_frame_encoded_processing_time_ms, |
| Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); |
| metrics_logger_->LogMetric( |
| "on_frame_pre_decode_processing_time_ms", test_label_, |
| analyzer_stats_.on_frame_pre_decode_processing_time_ms, |
| Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); |
| metrics_logger_->LogMetric( |
| "on_frame_decoded_processing_time_ms", test_label_, |
| analyzer_stats_.on_frame_decoded_processing_time_ms, |
| Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); |
| metrics_logger_->LogMetric( |
| "on_frame_rendered_processing_time_ms", test_label_, |
| analyzer_stats_.on_frame_rendered_processing_time_ms, |
| Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); |
| metrics_logger_->LogMetric( |
| "on_decoder_error_processing_time_ms", test_label_, |
| analyzer_stats_.on_decoder_error_processing_time_ms, |
| Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); |
| } |
| } |
| |
| void DefaultVideoQualityAnalyzer::ReportResults( |
| const InternalStatsKey& key, |
| const StreamStats& stats, |
| const FrameCounters& frame_counters) { |
| TimeDelta test_duration = Now() - start_time_; |
| std::string test_case_name = GetTestCaseName(ToMetricName(key)); |
| // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. |
| std::map<std::string, std::string> metric_metadata{ |
| {MetricMetadataKey::kPeerMetadataKey, peers_->name(key.sender)}, |
| {MetricMetadataKey::kVideoStreamMetadataKey, streams_.name(key.stream)}, |
| {MetricMetadataKey::kSenderMetadataKey, peers_->name(key.sender)}, |
| {MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)}, |
| {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}}; |
| |
| metrics_logger_->LogMetric( |
| "psnr_dB", test_case_name, stats.psnr, Unit::kUnitless, |
| ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric( |
| "ssim", test_case_name, stats.ssim, Unit::kUnitless, |
| ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric("transport_time", test_case_name, |
| stats.transport_time_ms, Unit::kMilliseconds, |
| ImprovementDirection::kSmallerIsBetter, |
| metric_metadata); |
| metrics_logger_->LogMetric( |
| "total_delay_incl_transport", test_case_name, |
| stats.total_delay_incl_transport_ms, Unit::kMilliseconds, |
| ImprovementDirection::kSmallerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric( |
| "time_between_rendered_frames", test_case_name, |
| stats.time_between_rendered_frames_ms, Unit::kMilliseconds, |
| ImprovementDirection::kSmallerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "harmonic_framerate", test_case_name, stats.harmonic_framerate_fps, |
| Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "encode_frame_rate", test_case_name, |
| stats.encode_frame_rate.IsEmpty() |
| ? 0 |
| : stats.encode_frame_rate.GetEventsPerSecond(), |
| Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric( |
| "encode_time", test_case_name, stats.encode_time_ms, Unit::kMilliseconds, |
| ImprovementDirection::kSmallerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric("time_between_freezes", test_case_name, |
| stats.time_between_freezes_ms, Unit::kMilliseconds, |
| ImprovementDirection::kBiggerIsBetter, |
| metric_metadata); |
| metrics_logger_->LogMetric("freeze_time_ms", test_case_name, |
| stats.freeze_time_ms, Unit::kMilliseconds, |
| ImprovementDirection::kSmallerIsBetter, |
| metric_metadata); |
| metrics_logger_->LogMetric( |
| "pixels_per_frame", test_case_name, stats.resolution_of_decoded_frame, |
| Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "min_psnr_dB", test_case_name, |
| stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless, |
| ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric( |
| "decode_time", test_case_name, stats.decode_time_ms, Unit::kMilliseconds, |
| ImprovementDirection::kSmallerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric( |
| "receive_to_render_time", test_case_name, stats.receive_to_render_time_ms, |
| Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, |
| metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount, |
| ImprovementDirection::kSmallerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "frames_in_flight", test_case_name, |
| frame_counters.captured - frame_counters.rendered - |
| frame_counters.dropped, |
| Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "rendered_frames", test_case_name, frame_counters.rendered, Unit::kCount, |
| ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric( |
| "max_skipped", test_case_name, stats.skipped_between_rendered, |
| Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata); |
| metrics_logger_->LogMetric( |
| "target_encode_bitrate", test_case_name, |
| stats.target_encode_bitrate / 1000, Unit::kKilobitsPerSecond, |
| ImprovementDirection::kNeitherIsBetter, metric_metadata); |
| for (const auto& [spatial_layer, qp] : stats.spatial_layers_qp) { |
| std::map<std::string, std::string> qp_metadata = metric_metadata; |
| qp_metadata[MetricMetadataKey::kSpatialLayerMetadataKey] = |
| std::to_string(spatial_layer); |
| metrics_logger_->LogMetric("qp_sl" + std::to_string(spatial_layer), |
| test_case_name, qp, Unit::kUnitless, |
| ImprovementDirection::kSmallerIsBetter, |
| std::move(qp_metadata)); |
| } |
| metrics_logger_->LogSingleValueMetric( |
| "actual_encode_bitrate", test_case_name, |
| static_cast<double>(stats.total_encoded_images_payload) / |
| test_duration.seconds<double>() * kBitsInByte / 1000, |
| Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, |
| metric_metadata); |
| |
| if (options_.report_detailed_frame_stats) { |
| metrics_logger_->LogSingleValueMetric( |
| "capture_frame_rate", test_case_name, |
| stats.capture_frame_rate.IsEmpty() |
| ? 0 |
| : stats.capture_frame_rate.GetEventsPerSecond(), |
| Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "num_encoded_frames", test_case_name, frame_counters.encoded, |
| Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "num_decoded_frames", test_case_name, frame_counters.decoded, |
| Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "num_send_key_frames", test_case_name, stats.num_send_key_frames, |
| Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| metrics_logger_->LogSingleValueMetric( |
| "num_recv_key_frames", test_case_name, stats.num_recv_key_frames, |
| Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); |
| |
| metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name, |
| stats.recv_key_frame_size_bytes, Unit::kCount, |
| ImprovementDirection::kBiggerIsBetter, |
| metric_metadata); |
| metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name, |
| stats.recv_delta_frame_size_bytes, Unit::kCount, |
| ImprovementDirection::kBiggerIsBetter, |
| metric_metadata); |
| } |
| } |
| |
| std::string DefaultVideoQualityAnalyzer::GetTestCaseName( |
| const std::string& stream_label) const { |
| return test_label_ + "/" + stream_label; |
| } |
| |
| Timestamp DefaultVideoQualityAnalyzer::Now() { |
| return clock_->CurrentTime(); |
| } |
| |
| StatsKey DefaultVideoQualityAnalyzer::ToStatsKey( |
| const InternalStatsKey& key) const { |
| return StatsKey(streams_.name(key.stream), peers_->name(key.receiver)); |
| } |
| |
| std::string DefaultVideoQualityAnalyzer::ToMetricName( |
| const InternalStatsKey& key) const { |
| const std::string& stream_label = streams_.name(key.stream); |
| if (peers_->GetKnownSize() <= 2 && key.sender != key.receiver) { |
| // TODO(titovartem): remove this special case. |
| return stream_label; |
| } |
| rtc::StringBuilder out; |
| out << stream_label << "_" << peers_->name(key.sender) << "_" |
| << peers_->name(key.receiver); |
| return out.str(); |
| } |
| |
| std::string DefaultVideoQualityAnalyzer::GetStreamLabelInternal( |
| uint16_t frame_id) const { |
| auto it = captured_frames_in_flight_.find(frame_id); |
| if (it != captured_frames_in_flight_.end()) { |
| return streams_.name(it->second.stream()); |
| } |
| for (auto hist_it = stream_to_frame_id_history_.begin(); |
| hist_it != stream_to_frame_id_history_.end(); ++hist_it) { |
| auto hist_set_it = hist_it->second.find(frame_id); |
| if (hist_set_it != hist_it->second.end()) { |
| return streams_.name(hist_it->first); |
| } |
| } |
| RTC_CHECK(false) << "Unknown frame_id=" << frame_id; |
| } |
| |
| double DefaultVideoQualityAnalyzer::GetCpuUsagePercent() const { |
| return cpu_measurer_.GetCpuUsagePercent(); |
| } |
| |
| std::map<std::string, std::vector<uint16_t>> |
| DefaultVideoQualityAnalyzer::GetStreamFrames() const { |
| MutexLock lock(&mutex_); |
| std::map<std::string, std::vector<uint16_t>> out; |
| for (auto entry_it : stream_to_frame_id_full_history_) { |
| out.insert({streams_.name(entry_it.first), entry_it.second}); |
| } |
| return out; |
| } |
| |
| } // namespace webrtc |
