| /* | |
| * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. | |
| * | |
| * Use of this source code is governed by a BSD-style license | |
| * that can be found in the LICENSE file in the root of the source | |
| * tree. An additional intellectual property rights grant can be found | |
| * in the file PATENTS. All contributing project authors may | |
| * be found in the AUTHORS file in the root of the source tree. | |
| */ | |
| #include "modules/video_coding/utility/quality_scaler.h" | |
| #include <cstddef> | |
| #include <cstdint> | |
| #include <memory> | |
| #include <optional> | |
| #include "api/field_trials_view.h" | |
| #include "api/sequence_checker.h" | |
| #include "api/task_queue/task_queue_base.h" | |
| #include "api/units/time_delta.h" | |
| #include "api/video_codecs/video_encoder.h" | |
| #include "rtc_base/checks.h" | |
| #include "rtc_base/experiments/quality_scaler_settings.h" | |
| #include "rtc_base/experiments/quality_scaling_experiment.h" | |
| #include "rtc_base/logging.h" | |
| #include "rtc_base/numerics/exp_filter.h" | |
| #include "rtc_base/weak_ptr.h" | |
| namespace webrtc { | |
| namespace { | |
| // Threshold constant used until first downscale (to permit fast rampup). | |
| const int kMeasureMs = 2000; | |
| const float kSamplePeriodScaleFactor = 2.5; | |
| const int kFramedropPercentThreshold = 60; | |
| const size_t kMinFramesNeededToScale = 2 * 30; | |
| } // namespace | |
| class QualityScaler::QpSmoother { | |
| public: | |
| explicit QpSmoother(float alpha) | |
| : alpha_(alpha), | |
| // The initial value of last_sample_ms doesn't matter since the smoother | |
| // will ignore the time delta for the first update. | |
| last_sample_ms_(0), | |
| smoother_(alpha) {} | |
| std::optional<int> GetAvg() const { | |
| float value = smoother_.filtered(); | |
| if (value == ExpFilter::kValueUndefined) { | |
| return std::nullopt; | |
| } | |
| return static_cast<int>(value); | |
| } | |
| void Add(float sample, int64_t time_sent_us) { | |
| int64_t now_ms = time_sent_us / 1000; | |
| smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample); | |
| last_sample_ms_ = now_ms; | |
| } | |
| void Reset() { smoother_.Reset(alpha_); } | |
| private: | |
| const float alpha_; | |
| int64_t last_sample_ms_; | |
| ExpFilter smoother_; | |
| }; | |
| // The QualityScaler checks for QP periodically by queuing CheckQpTasks. The | |
| // task will either run to completion and trigger a new task being queued, or it | |
| // will be destroyed because the QualityScaler is destroyed. | |
| // | |
| // When high or low QP is reported, the task will be pending until a callback is | |
| // invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage | |
| // asynchronously and prevents checking for QP until the stream has potentially | |
| // been reconfigured. | |
| class QualityScaler::CheckQpTask { | |
| public: | |
| // The result of one CheckQpTask may influence the delay of the next | |
| // CheckQpTask. | |
| struct Result { | |
| bool observed_enough_frames = false; | |
| bool qp_usage_reported = false; | |
| }; | |
| CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result) | |
| : quality_scaler_(quality_scaler), | |
| state_(State::kNotStarted), | |
| previous_task_result_(previous_task_result), | |
| weak_ptr_factory_(this) {} | |
| void StartDelayedTask() { | |
| RTC_DCHECK_EQ(state_, State::kNotStarted); | |
| state_ = State::kCheckingQp; | |
| TaskQueueBase::Current()->PostDelayedTask( | |
| [this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] { | |
| if (!this_weak_ptr) { | |
| // The task has been cancelled through destruction. | |
| return; | |
| } | |
| RTC_DCHECK_EQ(state_, State::kCheckingQp); | |
| RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_); | |
| switch (quality_scaler_->CheckQp()) { | |
| case QualityScaler::CheckQpResult::kInsufficientSamples: { | |
| result_.observed_enough_frames = false; | |
| // After this line, `this` may be deleted. | |
| break; | |
| } | |
| case QualityScaler::CheckQpResult::kNormalQp: { | |
| result_.observed_enough_frames = true; | |
| break; | |
| } | |
| case QualityScaler::CheckQpResult::kHighQp: { | |
| result_.observed_enough_frames = true; | |
| result_.qp_usage_reported = true; | |
| quality_scaler_->fast_rampup_ = false; | |
| quality_scaler_->handler_->OnReportQpUsageHigh(); | |
| quality_scaler_->ClearSamples(); | |
| break; | |
| } | |
| case QualityScaler::CheckQpResult::kLowQp: { | |
| result_.observed_enough_frames = true; | |
| result_.qp_usage_reported = true; | |
| quality_scaler_->handler_->OnReportQpUsageLow(); | |
| quality_scaler_->ClearSamples(); | |
| break; | |
| } | |
| } | |
| state_ = State::kCompleted; | |
| // Starting the next task deletes the pending task. After this line, | |
| // `this` has been deleted. | |
| quality_scaler_->StartNextCheckQpTask(); | |
| }, | |
| TimeDelta::Millis(GetCheckingQpDelayMs())); | |
| } | |
| bool HasCompletedTask() const { return state_ == State::kCompleted; } | |
| Result result() const { | |
| RTC_DCHECK(HasCompletedTask()); | |
| return result_; | |
| } | |
| private: | |
| enum class State { | |
| kNotStarted, | |
| kCheckingQp, | |
| kCompleted, | |
| }; | |
| // Determines the sampling period of CheckQpTasks. | |
| int64_t GetCheckingQpDelayMs() const { | |
| RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_); | |
| if (quality_scaler_->fast_rampup_) { | |
| return quality_scaler_->sampling_period_ms_; | |
| } | |
| if (quality_scaler_->experiment_enabled_ && | |
| !previous_task_result_.observed_enough_frames) { | |
| // Use half the interval while waiting for enough frames. | |
| return quality_scaler_->sampling_period_ms_ / 2; | |
| } | |
| if (quality_scaler_->scale_factor_ && | |
| !previous_task_result_.qp_usage_reported) { | |
| // Last CheckQp did not call AdaptDown/Up, possibly reduce interval. | |
| return quality_scaler_->sampling_period_ms_ * | |
| quality_scaler_->scale_factor_.value(); | |
| } | |
| return quality_scaler_->sampling_period_ms_ * | |
| quality_scaler_->initial_scale_factor_; | |
| } | |
| QualityScaler* const quality_scaler_; | |
| State state_; | |
| const Result previous_task_result_; | |
| Result result_; | |
| WeakPtrFactory<CheckQpTask> weak_ptr_factory_; | |
| }; | |
| QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler, | |
| VideoEncoder::QpThresholds thresholds, | |
| const FieldTrialsView& field_trials) | |
| : QualityScaler(handler, thresholds, field_trials, kMeasureMs) {} | |
| // Protected ctor, should not be called directly. | |
| QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler, | |
| VideoEncoder::QpThresholds thresholds, | |
| const FieldTrialsView& field_trials, | |
| int64_t default_sampling_period_ms) | |
| : handler_(handler), | |
| thresholds_(thresholds), | |
| sampling_period_ms_(QualityScalerSettings(field_trials) | |
| .SamplingPeriodMs() | |
| .value_or(default_sampling_period_ms)), | |
| fast_rampup_(true), | |
| // Arbitrarily choose size based on 30 fps for 5 seconds. | |
| average_qp_(QualityScalerSettings(field_trials) | |
| .AverageQpWindow() | |
| .value_or(5 * 30)), | |
| framedrop_percent_media_opt_(5 * 30), | |
| framedrop_percent_all_(5 * 30), | |
| experiment_enabled_(QualityScalingExperiment::Enabled(field_trials)), | |
| min_frames_needed_(QualityScalerSettings(field_trials) | |
| .MinFrames() | |
| .value_or(kMinFramesNeededToScale)), | |
| initial_scale_factor_(QualityScalerSettings(field_trials) | |
| .InitialScaleFactor() | |
| .value_or(kSamplePeriodScaleFactor)), | |
| scale_factor_(QualityScalerSettings(field_trials).ScaleFactor()) { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| if (experiment_enabled_) { | |
| config_ = QualityScalingExperiment::GetConfig(field_trials); | |
| qp_smoother_high_.reset(new QpSmoother(config_.alpha_high)); | |
| qp_smoother_low_.reset(new QpSmoother(config_.alpha_low)); | |
| } | |
| RTC_DCHECK(handler_ != nullptr); | |
| StartNextCheckQpTask(); | |
| RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low | |
| << ", high: " << thresholds_.high; | |
| } | |
| QualityScaler::~QualityScaler() { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| } | |
| void QualityScaler::StartNextCheckQpTask() { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask()) | |
| << "A previous CheckQpTask has not completed yet!"; | |
| CheckQpTask::Result previous_task_result; | |
| if (pending_qp_task_) { | |
| previous_task_result = pending_qp_task_->result(); | |
| } | |
| pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result); | |
| pending_qp_task_->StartDelayedTask(); | |
| } | |
| void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| thresholds_ = thresholds; | |
| } | |
| void QualityScaler::ReportDroppedFrameByMediaOpt() { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| framedrop_percent_media_opt_.AddSample(100); | |
| framedrop_percent_all_.AddSample(100); | |
| } | |
| void QualityScaler::ReportDroppedFrameByEncoder() { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| framedrop_percent_all_.AddSample(100); | |
| } | |
| void QualityScaler::ReportQp(int qp, int64_t time_sent_us) { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| framedrop_percent_media_opt_.AddSample(0); | |
| framedrop_percent_all_.AddSample(0); | |
| average_qp_.AddSample(qp); | |
| if (qp_smoother_high_) | |
| qp_smoother_high_->Add(qp, time_sent_us); | |
| if (qp_smoother_low_) | |
| qp_smoother_low_->Add(qp, time_sent_us); | |
| } | |
| QualityScaler::CheckQpResult QualityScaler::CheckQp() const { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| // Should be set through InitEncode -> Should be set by now. | |
| RTC_DCHECK_GE(thresholds_.low, 0); | |
| // If we have not observed at least this many frames we can't make a good | |
| // scaling decision. | |
| const size_t frames = config_.use_all_drop_reasons | |
| ? framedrop_percent_all_.Size() | |
| : framedrop_percent_media_opt_.Size(); | |
| if (frames < min_frames_needed_) { | |
| return CheckQpResult::kInsufficientSamples; | |
| } | |
| // Check if we should scale down due to high frame drop. | |
| const std::optional<int> drop_rate = | |
| config_.use_all_drop_reasons | |
| ? framedrop_percent_all_.GetAverageRoundedDown() | |
| : framedrop_percent_media_opt_.GetAverageRoundedDown(); | |
| if (drop_rate && *drop_rate >= kFramedropPercentThreshold) { | |
| RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate; | |
| return CheckQpResult::kHighQp; | |
| } | |
| // Check if we should scale up or down based on QP. | |
| const std::optional<int> avg_qp_high = | |
| qp_smoother_high_ ? qp_smoother_high_->GetAvg() | |
| : average_qp_.GetAverageRoundedDown(); | |
| const std::optional<int> avg_qp_low = | |
| qp_smoother_low_ ? qp_smoother_low_->GetAvg() | |
| : average_qp_.GetAverageRoundedDown(); | |
| if (avg_qp_high && avg_qp_low) { | |
| RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " (" | |
| << *avg_qp_low << ")."; | |
| if (*avg_qp_high > thresholds_.high) { | |
| return CheckQpResult::kHighQp; | |
| } | |
| if (*avg_qp_low <= thresholds_.low) { | |
| // QP has been low. We want to try a higher resolution. | |
| return CheckQpResult::kLowQp; | |
| } | |
| } | |
| return CheckQpResult::kNormalQp; | |
| } | |
| void QualityScaler::ClearSamples() { | |
| RTC_DCHECK_RUN_ON(&task_checker_); | |
| framedrop_percent_media_opt_.Reset(); | |
| framedrop_percent_all_.Reset(); | |
| average_qp_.Reset(); | |
| if (qp_smoother_high_) | |
| qp_smoother_high_->Reset(); | |
| if (qp_smoother_low_) | |
| qp_smoother_low_->Reset(); | |
| } | |
| QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {} | |
| } // namespace webrtc |