Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 1 | /* |
Tommi | 553c869 | 2020-05-05 13:35:45 | [diff] [blame] | 2 | * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include "video/video_quality_observer2.h" |
| 12 | |
| 13 | #include <algorithm> |
| 14 | #include <cmath> |
| 15 | #include <cstdint> |
| 16 | #include <string> |
| 17 | |
| 18 | #include "rtc_base/logging.h" |
| 19 | #include "rtc_base/strings/string_builder.h" |
| 20 | #include "system_wrappers/include/metrics.h" |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 21 | #include "video/video_receive_stream2.h" |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 22 | |
| 23 | namespace webrtc { |
| 24 | namespace internal { |
| 25 | const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5; |
| 26 | const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150; |
| 27 | const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30; |
| 28 | |
| 29 | namespace { |
| 30 | constexpr int kMinVideoDurationMs = 3000; |
| 31 | constexpr int kMinRequiredSamples = 1; |
| 32 | constexpr int kPixelsInHighResolution = |
| 33 | 960 * 540; // CPU-adapted HD still counts. |
| 34 | constexpr int kPixelsInMediumResolution = 640 * 360; |
| 35 | constexpr int kBlockyQpThresholdVp8 = 70; |
| 36 | constexpr int kBlockyQpThresholdVp9 = 180; |
| 37 | constexpr int kMaxNumCachedBlockyFrames = 100; |
| 38 | // TODO(ilnik): Add H264/HEVC thresholds. |
| 39 | } // namespace |
| 40 | |
Tommi | 553c869 | 2020-05-05 13:35:45 | [diff] [blame] | 41 | VideoQualityObserver::VideoQualityObserver() |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 42 | : last_frame_rendered_ms_(-1), |
| 43 | num_frames_rendered_(0), |
| 44 | first_frame_rendered_ms_(-1), |
| 45 | last_frame_pixels_(0), |
| 46 | is_last_frame_blocky_(false), |
| 47 | last_unfreeze_time_ms_(0), |
| 48 | render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames), |
| 49 | sum_squared_interframe_delays_secs_(0.0), |
| 50 | time_in_resolution_ms_(3, 0), |
| 51 | current_resolution_(Resolution::Low), |
| 52 | num_resolution_downgrades_(0), |
| 53 | time_in_blocky_video_ms_(0), |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 54 | is_paused_(false) {} |
| 55 | |
Tommi | 553c869 | 2020-05-05 13:35:45 | [diff] [blame] | 56 | void VideoQualityObserver::UpdateHistograms(bool screenshare) { |
| 57 | // TODO(bugs.webrtc.org/11489): Called on the decoder thread - which _might_ |
| 58 | // be the same as the construction thread. |
| 59 | |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 60 | // Don't report anything on an empty video stream. |
| 61 | if (num_frames_rendered_ == 0) { |
| 62 | return; |
| 63 | } |
| 64 | |
| 65 | char log_stream_buf[2 * 1024]; |
| 66 | rtc::SimpleStringBuilder log_stream(log_stream_buf); |
| 67 | |
| 68 | if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) { |
| 69 | smooth_playback_durations_.Add(last_frame_rendered_ms_ - |
| 70 | last_unfreeze_time_ms_); |
| 71 | } |
| 72 | |
Tommi | 553c869 | 2020-05-05 13:35:45 | [diff] [blame] | 73 | std::string uma_prefix = |
| 74 | screenshare ? "WebRTC.Video.Screenshare" : "WebRTC.Video"; |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 75 | |
| 76 | auto mean_time_between_freezes = |
| 77 | smooth_playback_durations_.Avg(kMinRequiredSamples); |
| 78 | if (mean_time_between_freezes) { |
| 79 | RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs", |
| 80 | *mean_time_between_freezes); |
| 81 | log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs " |
| 82 | << *mean_time_between_freezes << "\n"; |
| 83 | } |
| 84 | auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples); |
| 85 | if (avg_freeze_length) { |
| 86 | RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs", |
| 87 | *avg_freeze_length); |
| 88 | log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length |
| 89 | << "\n"; |
| 90 | } |
| 91 | |
| 92 | int64_t video_duration_ms = |
| 93 | last_frame_rendered_ms_ - first_frame_rendered_ms_; |
| 94 | |
| 95 | if (video_duration_ms >= kMinVideoDurationMs) { |
| 96 | int time_spent_in_hd_percentage = static_cast<int>( |
| 97 | time_in_resolution_ms_[Resolution::High] * 100 / video_duration_ms); |
| 98 | RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage", |
| 99 | time_spent_in_hd_percentage); |
| 100 | log_stream << uma_prefix << ".TimeInHdPercentage " |
| 101 | << time_spent_in_hd_percentage << "\n"; |
| 102 | |
| 103 | int time_with_blocky_video_percentage = |
| 104 | static_cast<int>(time_in_blocky_video_ms_ * 100 / video_duration_ms); |
| 105 | RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage", |
| 106 | time_with_blocky_video_percentage); |
| 107 | log_stream << uma_prefix << ".TimeInBlockyVideoPercentage " |
| 108 | << time_with_blocky_video_percentage << "\n"; |
| 109 | |
| 110 | int num_resolution_downgrades_per_minute = |
| 111 | num_resolution_downgrades_ * 60000 / video_duration_ms; |
Ilya Nikolaevskiy | a55a54d | 2022-12-08 10:00:21 | [diff] [blame] | 112 | if (!screenshare) { |
| 113 | RTC_HISTOGRAM_COUNTS_SPARSE_100( |
| 114 | uma_prefix + ".NumberResolutionDownswitchesPerMinute", |
| 115 | num_resolution_downgrades_per_minute); |
| 116 | log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute " |
| 117 | << num_resolution_downgrades_per_minute << "\n"; |
| 118 | } |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 119 | |
| 120 | int num_freezes_per_minute = |
| 121 | freezes_durations_.NumSamples() * 60000 / video_duration_ms; |
| 122 | RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".NumberFreezesPerMinute", |
| 123 | num_freezes_per_minute); |
| 124 | log_stream << uma_prefix << ".NumberFreezesPerMinute " |
| 125 | << num_freezes_per_minute << "\n"; |
| 126 | |
| 127 | if (sum_squared_interframe_delays_secs_ > 0.0) { |
| 128 | int harmonic_framerate_fps = std::round( |
| 129 | video_duration_ms / (1000 * sum_squared_interframe_delays_secs_)); |
| 130 | RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".HarmonicFrameRate", |
| 131 | harmonic_framerate_fps); |
| 132 | log_stream << uma_prefix << ".HarmonicFrameRate " |
| 133 | << harmonic_framerate_fps << "\n"; |
| 134 | } |
| 135 | } |
| 136 | RTC_LOG(LS_INFO) << log_stream.str(); |
| 137 | } |
| 138 | |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 139 | void VideoQualityObserver::OnRenderedFrame( |
| 140 | const VideoFrameMetaData& frame_meta) { |
| 141 | RTC_DCHECK_LE(last_frame_rendered_ms_, frame_meta.decode_timestamp.ms()); |
| 142 | RTC_DCHECK_LE(last_unfreeze_time_ms_, frame_meta.decode_timestamp.ms()); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 143 | |
| 144 | if (num_frames_rendered_ == 0) { |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 145 | first_frame_rendered_ms_ = last_unfreeze_time_ms_ = |
| 146 | frame_meta.decode_timestamp.ms(); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 147 | } |
| 148 | |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 149 | auto blocky_frame_it = blocky_frames_.find(frame_meta.rtp_timestamp); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 150 | |
| 151 | if (num_frames_rendered_ > 0) { |
| 152 | // Process inter-frame delay. |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 153 | const int64_t interframe_delay_ms = |
| 154 | frame_meta.decode_timestamp.ms() - last_frame_rendered_ms_; |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 155 | const double interframe_delays_secs = interframe_delay_ms / 1000.0; |
| 156 | |
| 157 | // Sum of squared inter frame intervals is used to calculate the harmonic |
| 158 | // frame rate metric. The metric aims to reflect overall experience related |
| 159 | // to smoothness of video playback and includes both freezes and pauses. |
| 160 | sum_squared_interframe_delays_secs_ += |
| 161 | interframe_delays_secs * interframe_delays_secs; |
| 162 | |
| 163 | if (!is_paused_) { |
| 164 | render_interframe_delays_.AddSample(interframe_delay_ms); |
| 165 | |
| 166 | bool was_freeze = false; |
| 167 | if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) { |
| 168 | const absl::optional<int64_t> avg_interframe_delay = |
| 169 | render_interframe_delays_.GetAverageRoundedDown(); |
| 170 | RTC_DCHECK(avg_interframe_delay); |
| 171 | was_freeze = interframe_delay_ms >= |
| 172 | std::max(3 * *avg_interframe_delay, |
| 173 | *avg_interframe_delay + kMinIncreaseForFreezeMs); |
| 174 | } |
| 175 | |
| 176 | if (was_freeze) { |
| 177 | freezes_durations_.Add(interframe_delay_ms); |
| 178 | smooth_playback_durations_.Add(last_frame_rendered_ms_ - |
| 179 | last_unfreeze_time_ms_); |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 180 | last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms(); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 181 | } else { |
| 182 | // Count spatial metrics if there were no freeze. |
| 183 | time_in_resolution_ms_[current_resolution_] += interframe_delay_ms; |
| 184 | |
| 185 | if (is_last_frame_blocky_) { |
| 186 | time_in_blocky_video_ms_ += interframe_delay_ms; |
| 187 | } |
| 188 | } |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | if (is_paused_) { |
| 193 | // If the stream was paused since the previous frame, do not count the |
| 194 | // pause toward smooth playback. Explicitly count the part before it and |
| 195 | // start the new smooth playback interval from this frame. |
| 196 | is_paused_ = false; |
| 197 | if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) { |
| 198 | smooth_playback_durations_.Add(last_frame_rendered_ms_ - |
| 199 | last_unfreeze_time_ms_); |
| 200 | } |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 201 | last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms(); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 202 | |
| 203 | if (num_frames_rendered_ > 0) { |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 204 | pauses_durations_.Add(frame_meta.decode_timestamp.ms() - |
| 205 | last_frame_rendered_ms_); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 206 | } |
| 207 | } |
| 208 | |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 209 | int64_t pixels = frame_meta.width * frame_meta.height; |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 210 | if (pixels >= kPixelsInHighResolution) { |
| 211 | current_resolution_ = Resolution::High; |
| 212 | } else if (pixels >= kPixelsInMediumResolution) { |
| 213 | current_resolution_ = Resolution::Medium; |
| 214 | } else { |
| 215 | current_resolution_ = Resolution::Low; |
| 216 | } |
| 217 | |
| 218 | if (pixels < last_frame_pixels_) { |
| 219 | ++num_resolution_downgrades_; |
| 220 | } |
| 221 | |
| 222 | last_frame_pixels_ = pixels; |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 223 | last_frame_rendered_ms_ = frame_meta.decode_timestamp.ms(); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 224 | |
| 225 | is_last_frame_blocky_ = blocky_frame_it != blocky_frames_.end(); |
| 226 | if (is_last_frame_blocky_) { |
| 227 | blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it); |
| 228 | } |
| 229 | |
| 230 | ++num_frames_rendered_; |
| 231 | } |
| 232 | |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 233 | void VideoQualityObserver::OnDecodedFrame(uint32_t rtp_frame_timestamp, |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 234 | absl::optional<uint8_t> qp, |
| 235 | VideoCodecType codec) { |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 236 | if (!qp) |
| 237 | return; |
| 238 | |
| 239 | absl::optional<int> qp_blocky_threshold; |
| 240 | // TODO(ilnik): add other codec types when we have QP for them. |
| 241 | switch (codec) { |
| 242 | case kVideoCodecVP8: |
| 243 | qp_blocky_threshold = kBlockyQpThresholdVp8; |
| 244 | break; |
| 245 | case kVideoCodecVP9: |
| 246 | qp_blocky_threshold = kBlockyQpThresholdVp9; |
| 247 | break; |
| 248 | default: |
| 249 | qp_blocky_threshold = absl::nullopt; |
| 250 | } |
| 251 | |
| 252 | RTC_DCHECK(blocky_frames_.find(rtp_frame_timestamp) == blocky_frames_.end()); |
| 253 | |
| 254 | if (qp_blocky_threshold && *qp > *qp_blocky_threshold) { |
| 255 | // Cache blocky frame. Its duration will be calculated in render callback. |
| 256 | if (blocky_frames_.size() > kMaxNumCachedBlockyFrames) { |
| 257 | RTC_LOG(LS_WARNING) << "Overflow of blocky frames cache."; |
| 258 | blocky_frames_.erase( |
| 259 | blocky_frames_.begin(), |
| 260 | std::next(blocky_frames_.begin(), kMaxNumCachedBlockyFrames / 2)); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 261 | } |
| 262 | |
Tommi | d7e08c8 | 2020-05-10 09:24:43 | [diff] [blame] | 263 | blocky_frames_.insert(rtp_frame_timestamp); |
Tommi | 74fc574 | 2020-04-27 08:43:06 | [diff] [blame] | 264 | } |
| 265 | } |
| 266 | |
| 267 | void VideoQualityObserver::OnStreamInactive() { |
| 268 | is_paused_ = true; |
| 269 | } |
| 270 | |
| 271 | uint32_t VideoQualityObserver::NumFreezes() const { |
| 272 | return freezes_durations_.NumSamples(); |
| 273 | } |
| 274 | |
| 275 | uint32_t VideoQualityObserver::NumPauses() const { |
| 276 | return pauses_durations_.NumSamples(); |
| 277 | } |
| 278 | |
| 279 | uint32_t VideoQualityObserver::TotalFreezesDurationMs() const { |
| 280 | return freezes_durations_.Sum(kMinRequiredSamples).value_or(0); |
| 281 | } |
| 282 | |
| 283 | uint32_t VideoQualityObserver::TotalPausesDurationMs() const { |
| 284 | return pauses_durations_.Sum(kMinRequiredSamples).value_or(0); |
| 285 | } |
| 286 | |
| 287 | uint32_t VideoQualityObserver::TotalFramesDurationMs() const { |
| 288 | return last_frame_rendered_ms_ - first_frame_rendered_ms_; |
| 289 | } |
| 290 | |
| 291 | double VideoQualityObserver::SumSquaredFrameDurationsSec() const { |
| 292 | return sum_squared_interframe_delays_secs_; |
| 293 | } |
| 294 | |
| 295 | } // namespace internal |
| 296 | } // namespace webrtc |