blob: 0afc2f5235a1997e6eb1597c2915a2b13f6bf358 [file] [log] [blame]
Tommi74fc5742020-04-27 08:43:061/*
Tommi553c8692020-05-05 13:35:452 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
Tommi74fc5742020-04-27 08:43:063 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "video/video_quality_observer2.h"
12
13#include <algorithm>
14#include <cmath>
15#include <cstdint>
16#include <string>
17
18#include "rtc_base/logging.h"
19#include "rtc_base/strings/string_builder.h"
20#include "system_wrappers/include/metrics.h"
Tommid7e08c82020-05-10 09:24:4321#include "video/video_receive_stream2.h"
Tommi74fc5742020-04-27 08:43:0622
23namespace webrtc {
24namespace internal {
25const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5;
26const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150;
27const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30;
28
29namespace {
30constexpr int kMinVideoDurationMs = 3000;
31constexpr int kMinRequiredSamples = 1;
32constexpr int kPixelsInHighResolution =
33 960 * 540; // CPU-adapted HD still counts.
34constexpr int kPixelsInMediumResolution = 640 * 360;
35constexpr int kBlockyQpThresholdVp8 = 70;
36constexpr int kBlockyQpThresholdVp9 = 180;
37constexpr int kMaxNumCachedBlockyFrames = 100;
38// TODO(ilnik): Add H264/HEVC thresholds.
39} // namespace
40
Tommi553c8692020-05-05 13:35:4541VideoQualityObserver::VideoQualityObserver()
Tommi74fc5742020-04-27 08:43:0642 : last_frame_rendered_ms_(-1),
43 num_frames_rendered_(0),
44 first_frame_rendered_ms_(-1),
45 last_frame_pixels_(0),
46 is_last_frame_blocky_(false),
47 last_unfreeze_time_ms_(0),
48 render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames),
49 sum_squared_interframe_delays_secs_(0.0),
50 time_in_resolution_ms_(3, 0),
51 current_resolution_(Resolution::Low),
52 num_resolution_downgrades_(0),
53 time_in_blocky_video_ms_(0),
Tommi74fc5742020-04-27 08:43:0654 is_paused_(false) {}
55
Tommi553c8692020-05-05 13:35:4556void VideoQualityObserver::UpdateHistograms(bool screenshare) {
57 // TODO(bugs.webrtc.org/11489): Called on the decoder thread - which _might_
58 // be the same as the construction thread.
59
Tommi74fc5742020-04-27 08:43:0660 // Don't report anything on an empty video stream.
61 if (num_frames_rendered_ == 0) {
62 return;
63 }
64
65 char log_stream_buf[2 * 1024];
66 rtc::SimpleStringBuilder log_stream(log_stream_buf);
67
68 if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
69 smooth_playback_durations_.Add(last_frame_rendered_ms_ -
70 last_unfreeze_time_ms_);
71 }
72
Tommi553c8692020-05-05 13:35:4573 std::string uma_prefix =
74 screenshare ? "WebRTC.Video.Screenshare" : "WebRTC.Video";
Tommi74fc5742020-04-27 08:43:0675
76 auto mean_time_between_freezes =
77 smooth_playback_durations_.Avg(kMinRequiredSamples);
78 if (mean_time_between_freezes) {
79 RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs",
80 *mean_time_between_freezes);
81 log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs "
82 << *mean_time_between_freezes << "\n";
83 }
84 auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples);
85 if (avg_freeze_length) {
86 RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs",
87 *avg_freeze_length);
88 log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length
89 << "\n";
90 }
91
92 int64_t video_duration_ms =
93 last_frame_rendered_ms_ - first_frame_rendered_ms_;
94
95 if (video_duration_ms >= kMinVideoDurationMs) {
96 int time_spent_in_hd_percentage = static_cast<int>(
97 time_in_resolution_ms_[Resolution::High] * 100 / video_duration_ms);
98 RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage",
99 time_spent_in_hd_percentage);
100 log_stream << uma_prefix << ".TimeInHdPercentage "
101 << time_spent_in_hd_percentage << "\n";
102
103 int time_with_blocky_video_percentage =
104 static_cast<int>(time_in_blocky_video_ms_ * 100 / video_duration_ms);
105 RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage",
106 time_with_blocky_video_percentage);
107 log_stream << uma_prefix << ".TimeInBlockyVideoPercentage "
108 << time_with_blocky_video_percentage << "\n";
109
110 int num_resolution_downgrades_per_minute =
111 num_resolution_downgrades_ * 60000 / video_duration_ms;
Ilya Nikolaevskiya55a54d2022-12-08 10:00:21112 if (!screenshare) {
113 RTC_HISTOGRAM_COUNTS_SPARSE_100(
114 uma_prefix + ".NumberResolutionDownswitchesPerMinute",
115 num_resolution_downgrades_per_minute);
116 log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute "
117 << num_resolution_downgrades_per_minute << "\n";
118 }
Tommi74fc5742020-04-27 08:43:06119
120 int num_freezes_per_minute =
121 freezes_durations_.NumSamples() * 60000 / video_duration_ms;
122 RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".NumberFreezesPerMinute",
123 num_freezes_per_minute);
124 log_stream << uma_prefix << ".NumberFreezesPerMinute "
125 << num_freezes_per_minute << "\n";
126
127 if (sum_squared_interframe_delays_secs_ > 0.0) {
128 int harmonic_framerate_fps = std::round(
129 video_duration_ms / (1000 * sum_squared_interframe_delays_secs_));
130 RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".HarmonicFrameRate",
131 harmonic_framerate_fps);
132 log_stream << uma_prefix << ".HarmonicFrameRate "
133 << harmonic_framerate_fps << "\n";
134 }
135 }
136 RTC_LOG(LS_INFO) << log_stream.str();
137}
138
Tommid7e08c82020-05-10 09:24:43139void VideoQualityObserver::OnRenderedFrame(
140 const VideoFrameMetaData& frame_meta) {
141 RTC_DCHECK_LE(last_frame_rendered_ms_, frame_meta.decode_timestamp.ms());
142 RTC_DCHECK_LE(last_unfreeze_time_ms_, frame_meta.decode_timestamp.ms());
Tommi74fc5742020-04-27 08:43:06143
144 if (num_frames_rendered_ == 0) {
Tommid7e08c82020-05-10 09:24:43145 first_frame_rendered_ms_ = last_unfreeze_time_ms_ =
146 frame_meta.decode_timestamp.ms();
Tommi74fc5742020-04-27 08:43:06147 }
148
Tommid7e08c82020-05-10 09:24:43149 auto blocky_frame_it = blocky_frames_.find(frame_meta.rtp_timestamp);
Tommi74fc5742020-04-27 08:43:06150
151 if (num_frames_rendered_ > 0) {
152 // Process inter-frame delay.
Tommid7e08c82020-05-10 09:24:43153 const int64_t interframe_delay_ms =
154 frame_meta.decode_timestamp.ms() - last_frame_rendered_ms_;
Tommi74fc5742020-04-27 08:43:06155 const double interframe_delays_secs = interframe_delay_ms / 1000.0;
156
157 // Sum of squared inter frame intervals is used to calculate the harmonic
158 // frame rate metric. The metric aims to reflect overall experience related
159 // to smoothness of video playback and includes both freezes and pauses.
160 sum_squared_interframe_delays_secs_ +=
161 interframe_delays_secs * interframe_delays_secs;
162
163 if (!is_paused_) {
164 render_interframe_delays_.AddSample(interframe_delay_ms);
165
166 bool was_freeze = false;
167 if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) {
168 const absl::optional<int64_t> avg_interframe_delay =
169 render_interframe_delays_.GetAverageRoundedDown();
170 RTC_DCHECK(avg_interframe_delay);
171 was_freeze = interframe_delay_ms >=
172 std::max(3 * *avg_interframe_delay,
173 *avg_interframe_delay + kMinIncreaseForFreezeMs);
174 }
175
176 if (was_freeze) {
177 freezes_durations_.Add(interframe_delay_ms);
178 smooth_playback_durations_.Add(last_frame_rendered_ms_ -
179 last_unfreeze_time_ms_);
Tommid7e08c82020-05-10 09:24:43180 last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms();
Tommi74fc5742020-04-27 08:43:06181 } else {
182 // Count spatial metrics if there were no freeze.
183 time_in_resolution_ms_[current_resolution_] += interframe_delay_ms;
184
185 if (is_last_frame_blocky_) {
186 time_in_blocky_video_ms_ += interframe_delay_ms;
187 }
188 }
189 }
190 }
191
192 if (is_paused_) {
193 // If the stream was paused since the previous frame, do not count the
194 // pause toward smooth playback. Explicitly count the part before it and
195 // start the new smooth playback interval from this frame.
196 is_paused_ = false;
197 if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
198 smooth_playback_durations_.Add(last_frame_rendered_ms_ -
199 last_unfreeze_time_ms_);
200 }
Tommid7e08c82020-05-10 09:24:43201 last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms();
Tommi74fc5742020-04-27 08:43:06202
203 if (num_frames_rendered_ > 0) {
Tommid7e08c82020-05-10 09:24:43204 pauses_durations_.Add(frame_meta.decode_timestamp.ms() -
205 last_frame_rendered_ms_);
Tommi74fc5742020-04-27 08:43:06206 }
207 }
208
Tommid7e08c82020-05-10 09:24:43209 int64_t pixels = frame_meta.width * frame_meta.height;
Tommi74fc5742020-04-27 08:43:06210 if (pixels >= kPixelsInHighResolution) {
211 current_resolution_ = Resolution::High;
212 } else if (pixels >= kPixelsInMediumResolution) {
213 current_resolution_ = Resolution::Medium;
214 } else {
215 current_resolution_ = Resolution::Low;
216 }
217
218 if (pixels < last_frame_pixels_) {
219 ++num_resolution_downgrades_;
220 }
221
222 last_frame_pixels_ = pixels;
Tommid7e08c82020-05-10 09:24:43223 last_frame_rendered_ms_ = frame_meta.decode_timestamp.ms();
Tommi74fc5742020-04-27 08:43:06224
225 is_last_frame_blocky_ = blocky_frame_it != blocky_frames_.end();
226 if (is_last_frame_blocky_) {
227 blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it);
228 }
229
230 ++num_frames_rendered_;
231}
232
Tommid7e08c82020-05-10 09:24:43233void VideoQualityObserver::OnDecodedFrame(uint32_t rtp_frame_timestamp,
Tommi74fc5742020-04-27 08:43:06234 absl::optional<uint8_t> qp,
235 VideoCodecType codec) {
Tommid7e08c82020-05-10 09:24:43236 if (!qp)
237 return;
238
239 absl::optional<int> qp_blocky_threshold;
240 // TODO(ilnik): add other codec types when we have QP for them.
241 switch (codec) {
242 case kVideoCodecVP8:
243 qp_blocky_threshold = kBlockyQpThresholdVp8;
244 break;
245 case kVideoCodecVP9:
246 qp_blocky_threshold = kBlockyQpThresholdVp9;
247 break;
248 default:
249 qp_blocky_threshold = absl::nullopt;
250 }
251
252 RTC_DCHECK(blocky_frames_.find(rtp_frame_timestamp) == blocky_frames_.end());
253
254 if (qp_blocky_threshold && *qp > *qp_blocky_threshold) {
255 // Cache blocky frame. Its duration will be calculated in render callback.
256 if (blocky_frames_.size() > kMaxNumCachedBlockyFrames) {
257 RTC_LOG(LS_WARNING) << "Overflow of blocky frames cache.";
258 blocky_frames_.erase(
259 blocky_frames_.begin(),
260 std::next(blocky_frames_.begin(), kMaxNumCachedBlockyFrames / 2));
Tommi74fc5742020-04-27 08:43:06261 }
262
Tommid7e08c82020-05-10 09:24:43263 blocky_frames_.insert(rtp_frame_timestamp);
Tommi74fc5742020-04-27 08:43:06264 }
265}
266
267void VideoQualityObserver::OnStreamInactive() {
268 is_paused_ = true;
269}
270
271uint32_t VideoQualityObserver::NumFreezes() const {
272 return freezes_durations_.NumSamples();
273}
274
275uint32_t VideoQualityObserver::NumPauses() const {
276 return pauses_durations_.NumSamples();
277}
278
279uint32_t VideoQualityObserver::TotalFreezesDurationMs() const {
280 return freezes_durations_.Sum(kMinRequiredSamples).value_or(0);
281}
282
283uint32_t VideoQualityObserver::TotalPausesDurationMs() const {
284 return pauses_durations_.Sum(kMinRequiredSamples).value_or(0);
285}
286
287uint32_t VideoQualityObserver::TotalFramesDurationMs() const {
288 return last_frame_rendered_ms_ - first_frame_rendered_ms_;
289}
290
291double VideoQualityObserver::SumSquaredFrameDurationsSec() const {
292 return sum_squared_interframe_delays_secs_;
293}
294
295} // namespace internal
296} // namespace webrtc