video/video_quality_observer2.cc

/*
 *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "video/video_quality_observer2.h"

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <iterator>
#include <optional>
#include <string>

#include "api/video/video_codec_type.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "system_wrappers/include/metrics.h"
#include "video/video_receive_stream2.h"

namespace webrtc {
namespace internal {
const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5;
const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150;
const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30;

namespace {
constexpr int kMinVideoDurationMs = 3000;
constexpr int kMinRequiredSamples = 1;
constexpr int kPixelsInHighResolution =
    960 * 540;  // CPU-adapted HD still counts.
constexpr int kPixelsInMediumResolution = 640 * 360;
constexpr int kBlockyQpThresholdVp8 = 70;
constexpr int kBlockyQpThresholdVp9 = 180;
constexpr int kMaxNumCachedBlockyFrames = 100;
// TODO(ilnik): Add H264/HEVC thresholds.
}  // namespace

VideoQualityObserver::VideoQualityObserver()
    : last_frame_rendered_ms_(-1),
      num_frames_rendered_(0),
      first_frame_rendered_ms_(-1),
      last_frame_pixels_(0),
      is_last_frame_blocky_(false),
      last_unfreeze_time_ms_(0),
      render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames),
      sum_squared_interframe_delays_secs_(0.0),
      time_in_resolution_ms_(3, 0),
      current_resolution_(Resolution::Low),
      num_resolution_downgrades_(0),
      time_in_blocky_video_ms_(0),
      is_paused_(false) {}

void VideoQualityObserver::UpdateHistograms(bool screenshare) {
  // TODO(bugs.webrtc.org/11489): Called on the decoder thread - which _might_
  // be the same as the construction thread.

  // Don't report anything on an empty video stream.
  if (num_frames_rendered_ == 0) {
    return;
  }

  char log_stream_buf[2 * 1024];
  SimpleStringBuilder log_stream(log_stream_buf);

  if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
    smooth_playback_durations_.Add(last_frame_rendered_ms_ -
                                   last_unfreeze_time_ms_);
  }

  std::string uma_prefix =
      screenshare ? "WebRTC.Video.Screenshare" : "WebRTC.Video";

  auto mean_time_between_freezes =
      smooth_playback_durations_.Avg(kMinRequiredSamples);
  if (mean_time_between_freezes) {
    RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs",
                                       *mean_time_between_freezes);
    log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs "
               << *mean_time_between_freezes << "\n";
  }
  auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples);
  if (avg_freeze_length) {
    RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs",
                                       *avg_freeze_length);
    log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length
               << "\n";
  }

  int64_t video_duration_ms =
      last_frame_rendered_ms_ - first_frame_rendered_ms_;

  if (video_duration_ms >= kMinVideoDurationMs) {
    int time_spent_in_hd_percentage = static_cast<int>(
        time_in_resolution_ms_[Resolution::High] * 100 / video_duration_ms);
    RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage",
                                    time_spent_in_hd_percentage);
    log_stream << uma_prefix << ".TimeInHdPercentage "
               << time_spent_in_hd_percentage << "\n";

    int time_with_blocky_video_percentage =
        static_cast<int>(time_in_blocky_video_ms_ * 100 / video_duration_ms);
    RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage",
                                    time_with_blocky_video_percentage);
    log_stream << uma_prefix << ".TimeInBlockyVideoPercentage "
               << time_with_blocky_video_percentage << "\n";

    int num_resolution_downgrades_per_minute =
        num_resolution_downgrades_ * 60000 / video_duration_ms;
    if (!screenshare) {
      RTC_HISTOGRAM_COUNTS_SPARSE_100(
          uma_prefix + ".NumberResolutionDownswitchesPerMinute",
          num_resolution_downgrades_per_minute);
      log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute "
                 << num_resolution_downgrades_per_minute << "\n";
    }

    int num_freezes_per_minute =
        freezes_durations_.NumSamples() * 60000 / video_duration_ms;
    RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".NumberFreezesPerMinute",
                                    num_freezes_per_minute);
    log_stream << uma_prefix << ".NumberFreezesPerMinute "
               << num_freezes_per_minute << "\n";

    if (sum_squared_interframe_delays_secs_ > 0.0) {
      int harmonic_framerate_fps = std::round(
          video_duration_ms / (1000 * sum_squared_interframe_delays_secs_));
      RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".HarmonicFrameRate",
                                      harmonic_framerate_fps);
      log_stream << uma_prefix << ".HarmonicFrameRate "
                 << harmonic_framerate_fps << "\n";
    }
  }
  RTC_LOG(LS_INFO) << log_stream.str();
}

void VideoQualityObserver::OnRenderedFrame(
    const VideoFrameMetaData& frame_meta) {
  RTC_DCHECK_LE(last_frame_rendered_ms_, frame_meta.decode_timestamp.ms());
  RTC_DCHECK_LE(last_unfreeze_time_ms_, frame_meta.decode_timestamp.ms());

  if (num_frames_rendered_ == 0) {
    first_frame_rendered_ms_ = last_unfreeze_time_ms_ =
        frame_meta.decode_timestamp.ms();
  }

  auto blocky_frame_it = blocky_frames_.find(frame_meta.rtp_timestamp);

  if (num_frames_rendered_ > 0) {
    // Process inter-frame delay.
    const int64_t interframe_delay_ms =
        frame_meta.decode_timestamp.ms() - last_frame_rendered_ms_;
    const double interframe_delays_secs = interframe_delay_ms / 1000.0;

    // Sum of squared inter frame intervals is used to calculate the harmonic
    // frame rate metric. The metric aims to reflect overall experience related
    // to smoothness of video playback and includes both freezes and pauses.
    sum_squared_interframe_delays_secs_ +=
        interframe_delays_secs * interframe_delays_secs;

    if (!is_paused_) {
      render_interframe_delays_.AddSample(interframe_delay_ms);

      bool was_freeze = false;
      if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) {
        const std::optional<int64_t> avg_interframe_delay =
            render_interframe_delays_.GetAverageRoundedDown();
        RTC_DCHECK(avg_interframe_delay);
        was_freeze = interframe_delay_ms >=
                     std::max(3 * *avg_interframe_delay,
                              *avg_interframe_delay + kMinIncreaseForFreezeMs);
      }

      if (was_freeze) {
        freezes_durations_.Add(interframe_delay_ms);
        smooth_playback_durations_.Add(last_frame_rendered_ms_ -
                                       last_unfreeze_time_ms_);
        last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms();
      } else {
        // Count spatial metrics if there were no freeze.
        time_in_resolution_ms_[current_resolution_] += interframe_delay_ms;

        if (is_last_frame_blocky_) {
          time_in_blocky_video_ms_ += interframe_delay_ms;
        }
      }
    }
  }

  if (is_paused_) {
    // If the stream was paused since the previous frame, do not count the
    // pause toward smooth playback. Explicitly count the part before it and
    // start the new smooth playback interval from this frame.
    is_paused_ = false;
    if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
      smooth_playback_durations_.Add(last_frame_rendered_ms_ -
                                     last_unfreeze_time_ms_);
    }
    last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms();

    if (num_frames_rendered_ > 0) {
      pauses_durations_.Add(frame_meta.decode_timestamp.ms() -
                            last_frame_rendered_ms_);
    }
  }

  int64_t pixels = frame_meta.width * frame_meta.height;
  if (pixels >= kPixelsInHighResolution) {
    current_resolution_ = Resolution::High;
  } else if (pixels >= kPixelsInMediumResolution) {
    current_resolution_ = Resolution::Medium;
  } else {
    current_resolution_ = Resolution::Low;
  }

  if (pixels < last_frame_pixels_) {
    ++num_resolution_downgrades_;
  }

  last_frame_pixels_ = pixels;
  last_frame_rendered_ms_ = frame_meta.decode_timestamp.ms();

  is_last_frame_blocky_ = blocky_frame_it != blocky_frames_.end();
  if (is_last_frame_blocky_) {
    blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it);
  }

  ++num_frames_rendered_;
}

void VideoQualityObserver::OnDecodedFrame(uint32_t rtp_frame_timestamp,
                                          std::optional<uint8_t> qp,
                                          VideoCodecType codec) {
  if (!qp)
    return;

  std::optional<int> qp_blocky_threshold;
  // TODO(ilnik): add other codec types when we have QP for them.
  switch (codec) {
    case kVideoCodecVP8:
      qp_blocky_threshold = kBlockyQpThresholdVp8;
      break;
    case kVideoCodecVP9:
      qp_blocky_threshold = kBlockyQpThresholdVp9;
      break;
    default:
      qp_blocky_threshold = std::nullopt;
  }

  RTC_DCHECK(blocky_frames_.find(rtp_frame_timestamp) == blocky_frames_.end());

  if (qp_blocky_threshold && *qp > *qp_blocky_threshold) {
    // Cache blocky frame. Its duration will be calculated in render callback.
    if (blocky_frames_.size() > kMaxNumCachedBlockyFrames) {
      RTC_LOG(LS_WARNING) << "Overflow of blocky frames cache.";
      blocky_frames_.erase(
          blocky_frames_.begin(),
          std::next(blocky_frames_.begin(), kMaxNumCachedBlockyFrames / 2));
    }

    blocky_frames_.insert(rtp_frame_timestamp);
  }
}

void VideoQualityObserver::OnStreamInactive() {
  is_paused_ = true;
}

uint32_t VideoQualityObserver::NumFreezes() const {
  return freezes_durations_.NumSamples();
}

uint32_t VideoQualityObserver::NumPauses() const {
  return pauses_durations_.NumSamples();
}

uint32_t VideoQualityObserver::TotalFreezesDurationMs() const {
  return freezes_durations_.Sum(kMinRequiredSamples).value_or(0);
}

uint32_t VideoQualityObserver::TotalPausesDurationMs() const {
  return pauses_durations_.Sum(kMinRequiredSamples).value_or(0);
}

uint32_t VideoQualityObserver::TotalFramesDurationMs() const {
  return last_frame_rendered_ms_ - first_frame_rendered_ms_;
}

double VideoQualityObserver::SumSquaredFrameDurationsSec() const {
  return sum_squared_interframe_delays_secs_;
}

}  // namespace internal
}  // namespace webrtc