Replace AdaptCount with a single counter.
There is still a counter for the active counts for the
scaling, but these will be removed at a later date.
BUG=webrtc:11392
Change-Id: Ie9bcf3f744a0bbac601f0da61197f4bac1e9f879
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169447
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Commit-Queue: Evan Shrubsole <eshr@google.com>
Cr-Commit-Position: refs/heads/master@{#30701}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 2f7ec17..a3e7d07 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -504,6 +504,7 @@
"end_to_end_tests/stats_tests.cc",
"end_to_end_tests/transport_feedback_tests.cc",
"frame_encode_metadata_writer_unittest.cc",
+ "overuse_frame_detector_resource_adaptation_unittest.cc",
"overuse_frame_detector_unittest.cc",
"picture_id_tests.cc",
"quality_limitation_reason_tracker_unittest.cc",
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
index 1b88bb3..792abc5 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.cc
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -67,8 +67,57 @@
return source_restrictions;
}
+// Returns AdaptationCounters where constraints that don't apply to the
+// degradation preference are cleared. This behaviour must reflect that of
+// ApplyDegradationPreference for SourceRestrictions. Any changed to that
+// method must also change this one.
+AdaptationCounters ApplyDegradationPreference(
+ AdaptationCounters counters,
+ DegradationPreference degradation_preference) {
+ switch (degradation_preference) {
+ case DegradationPreference::BALANCED:
+ break;
+ case DegradationPreference::MAINTAIN_FRAMERATE:
+ counters.fps_adaptations = 0;
+ break;
+ case DegradationPreference::MAINTAIN_RESOLUTION:
+ counters.resolution_adaptations = 0;
+ break;
+ case DegradationPreference::DISABLED:
+ counters.resolution_adaptations = 0;
+ counters.fps_adaptations = 0;
+ break;
+ default:
+ RTC_NOTREACHED();
+ }
+ return counters;
+}
+
} // namespace
+bool AdaptationCounters::operator==(const AdaptationCounters& rhs) const {
+ return fps_adaptations == rhs.fps_adaptations &&
+ resolution_adaptations == rhs.resolution_adaptations;
+}
+
+bool AdaptationCounters::operator!=(const AdaptationCounters& rhs) const {
+ return !(rhs == *this);
+}
+
+AdaptationCounters AdaptationCounters::operator+(
+ const AdaptationCounters& other) const {
+ return AdaptationCounters(
+ resolution_adaptations + other.resolution_adaptations,
+ fps_adaptations + other.fps_adaptations);
+}
+
+AdaptationCounters AdaptationCounters::operator-(
+ const AdaptationCounters& other) const {
+ return AdaptationCounters(
+ resolution_adaptations - other.resolution_adaptations,
+ fps_adaptations - other.fps_adaptations);
+}
+
// VideoSourceRestrictor is responsible for keeping track of current
// VideoSourceRestrictions and how to modify them in response to adapting up or
// down. It is not reponsible for determining when we should adapt up or down -
@@ -115,8 +164,10 @@
VideoSourceRestrictions source_restrictions() {
return source_restrictions_;
}
+ const AdaptationCounters& adaptation_counters() const { return adaptations_; }
void ClearRestrictions() {
source_restrictions_ = VideoSourceRestrictions();
+ adaptations_ = AdaptationCounters();
}
bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) {
@@ -135,6 +186,7 @@
? absl::optional<size_t>(target_pixels)
: absl::nullopt);
source_restrictions_.set_target_pixels_per_frame(absl::nullopt);
+ ++adaptations_.resolution_adaptations;
}
bool CanIncreaseResolutionTo(int target_pixels) {
@@ -157,6 +209,8 @@
max_pixels_wanted != std::numeric_limits<int>::max()
? absl::optional<size_t>(target_pixels)
: absl::nullopt);
+ --adaptations_.resolution_adaptations;
+ RTC_DCHECK_GE(adaptations_.resolution_adaptations, 0);
}
bool CanDecreaseFrameRateTo(int max_frame_rate) {
@@ -173,6 +227,7 @@
max_frame_rate != std::numeric_limits<int>::max()
? absl::optional<double>(max_frame_rate)
: absl::nullopt);
+ ++adaptations_.fps_adaptations;
}
bool CanIncreaseFrameRateTo(int max_frame_rate) {
@@ -187,6 +242,8 @@
max_frame_rate != std::numeric_limits<int>::max()
? absl::optional<double>(max_frame_rate)
: absl::nullopt);
+ --adaptations_.fps_adaptations;
+ RTC_DCHECK_GE(adaptations_.fps_adaptations, 0);
}
private:
@@ -207,113 +264,11 @@
}
VideoSourceRestrictions source_restrictions_;
+ AdaptationCounters adaptations_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
};
-class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final {
- public:
- AdaptCounter() {
- fps_counters_.resize(AdaptationObserverInterface::kScaleReasonSize);
- resolution_counters_.resize(AdaptationObserverInterface::kScaleReasonSize);
- static_assert(AdaptationObserverInterface::kScaleReasonSize == 2,
- "Update MoveCount.");
- }
- ~AdaptCounter() = default;
-
- // Get number of adaptation downscales for |reason|.
- VideoStreamEncoderObserver::AdaptationSteps Counts(int reason) const {
- VideoStreamEncoderObserver::AdaptationSteps counts;
- counts.num_framerate_reductions = fps_counters_[reason];
- counts.num_resolution_reductions = resolution_counters_[reason];
- return counts;
- }
-
- std::string ToString() const {
- rtc::StringBuilder ss;
- ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
- ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
- return ss.Release();
- }
-
- void IncrementFramerate(int reason) { ++(fps_counters_[reason]); }
- void IncrementResolution(int reason) { ++(resolution_counters_[reason]); }
- void DecrementFramerate(int reason) {
- if (fps_counters_[reason] == 0) {
- // Balanced mode: Adapt up is in a different order, switch reason.
- // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3).
- // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0}
- // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0}
- // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0}
- // 4. Up resolution (quality):res={quality:0,cpu:0}, fps={quality:0,cpu:0}
- RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
- RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
- MoveCount(&resolution_counters_, reason);
- MoveCount(&fps_counters_,
- (reason + 1) % AdaptationObserverInterface::kScaleReasonSize);
- }
- --(fps_counters_[reason]);
- RTC_DCHECK_GE(fps_counters_[reason], 0);
- }
-
- void DecrementResolution(int reason) {
- if (resolution_counters_[reason] == 0) {
- // Balanced mode: Adapt up is in a different order, switch reason.
- RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
- RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
- MoveCount(&fps_counters_, reason);
- MoveCount(&resolution_counters_,
- (reason + 1) % AdaptationObserverInterface::kScaleReasonSize);
- }
- --(resolution_counters_[reason]);
- RTC_DCHECK_GE(resolution_counters_[reason], 0);
- }
-
- void DecrementFramerate(int reason, int cur_fps) {
- DecrementFramerate(reason);
- // Reset if at max fps (i.e. in case of fewer steps up than down).
- if (cur_fps == std::numeric_limits<int>::max())
- absl::c_fill(fps_counters_, 0);
- }
-
- // Gets the total number of downgrades (for all adapt reasons).
- int FramerateCount() const { return Count(fps_counters_); }
- int ResolutionCount() const { return Count(resolution_counters_); }
-
- // Gets the total number of downgrades for |reason|.
- int FramerateCount(int reason) const { return fps_counters_[reason]; }
- int ResolutionCount(int reason) const { return resolution_counters_[reason]; }
- int TotalCount(int reason) const {
- return FramerateCount(reason) + ResolutionCount(reason);
- }
-
- private:
- std::string ToString(const std::vector<int>& counters) const {
- rtc::StringBuilder ss;
- for (size_t reason = 0;
- reason < AdaptationObserverInterface::kScaleReasonSize; ++reason) {
- ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
- }
- return ss.Release();
- }
-
- int Count(const std::vector<int>& counters) const {
- return absl::c_accumulate(counters, 0);
- }
-
- void MoveCount(std::vector<int>* counters, int from_reason) {
- int to_reason =
- (from_reason + 1) % AdaptationObserverInterface::kScaleReasonSize;
- ++((*counters)[to_reason]);
- --((*counters)[from_reason]);
- }
-
- // Degradation counters holding number of framerate/resolution reductions
- // per adapt reason.
- std::vector<int> fps_counters_;
- std::vector<int> resolution_counters_;
-};
-
class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper {
public:
explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource)
@@ -402,7 +357,6 @@
experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
has_input_video_(false),
degradation_preference_(DegradationPreference::DISABLED),
- adapt_counters_(),
balanced_settings_(),
last_adaptation_request_(absl::nullopt),
source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
@@ -418,10 +372,10 @@
quality_rampup_done_(false),
quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()),
encoder_settings_(absl::nullopt),
- encoder_stats_observer_(encoder_stats_observer) {
+ encoder_stats_observer_(encoder_stats_observer),
+ active_counts_() {
RTC_DCHECK(adaptation_listener_);
RTC_DCHECK(encoder_stats_observer_);
- ClearAdaptCounters();
AddResource(encode_usage_resource_.get(),
AdaptationObserverInterface::AdaptReason::kCpu);
AddResource(quality_scaler_resource_.get(),
@@ -488,10 +442,8 @@
last_adaptation_request_.reset();
if (degradation_preference == DegradationPreference::BALANCED ||
degradation_preference_ == DegradationPreference::BALANCED) {
- // TODO(asapersson): Consider removing |adapt_counters_| map and use one
- // AdaptCounter for all modes.
source_restrictor_->ClearRestrictions();
- ClearAdaptCounters();
+ active_counts_.fill(AdaptationCounters());
}
}
degradation_preference_ = degradation_preference;
@@ -533,7 +485,7 @@
ResetVideoSourceRestrictions() {
last_adaptation_request_.reset();
source_restrictor_->ClearRestrictions();
- ClearAdaptCounters();
+ active_counts_.fill(AdaptationCounters());
MaybeUpdateVideoSourceRestrictions();
}
@@ -543,19 +495,17 @@
}
void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() {
- int fps_count = GetConstAdaptCounter().FramerateCount(
- AdaptationObserverInterface::AdaptReason::kQuality);
- int res_count = GetConstAdaptCounter().ResolutionCount(
- AdaptationObserverInterface::AdaptReason::kQuality);
+ AdaptationCounters counters_before =
+ source_restrictor_->adaptation_counters();
OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
if (degradation_preference() == DegradationPreference::BALANCED &&
- GetConstAdaptCounter().FramerateCount(
- AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) {
+ source_restrictor_->adaptation_counters().fps_adaptations >
+ counters_before.fps_adaptations) {
// Adapt framerate in same step as resolution.
OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
}
- if (GetConstAdaptCounter().ResolutionCount(
- AdaptationObserverInterface::AdaptReason::kQuality) > res_count) {
+ if (source_restrictor_->adaptation_counters().resolution_adaptations >
+ counters_before.resolution_adaptations) {
encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
}
initial_frame_dropper_->OnFrameDroppedDueToSize();
@@ -690,7 +640,14 @@
if (!has_input_video_)
return absl::nullopt;
// 1. We can't adapt up if we're already at the highest setting.
- int num_downgrades = GetConstAdaptCounter().TotalCount(reason);
+ // Note that this only includes counts relevant to the current degradation
+ // preference. e.g. we previously adapted resolution, now prefer adpating fps,
+ // only count the fps adaptations and not the previous resolution adaptations.
+ // TODO(https://crbug.com/webrtc/11394): Checking the counts for reason should
+ // be replaced with checking the overuse state of all resources.
+ int num_downgrades = ApplyDegradationPreference(active_counts_[reason],
+ degradation_preference_)
+ .Total();
RTC_DCHECK_GE(num_downgrades, 0);
if (num_downgrades == 0)
return absl::nullopt;
@@ -745,7 +702,8 @@
}
// Attempt to increase pixel count.
int target_pixels = input_pixels;
- if (GetConstAdaptCounter().ResolutionCount() == 1) {
+ if (source_restrictor_->adaptation_counters().resolution_adaptations ==
+ 1) {
RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
target_pixels = std::numeric_limits<int>::max();
}
@@ -759,7 +717,7 @@
case DegradationPreference::MAINTAIN_RESOLUTION: {
// Scale up framerate.
int target_fps = input_fps;
- if (GetConstAdaptCounter().FramerateCount() == 1) {
+ if (source_restrictor_->adaptation_counters().fps_adaptations == 1) {
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
target_fps = std::numeric_limits<int>::max();
}
@@ -777,8 +735,7 @@
OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget(
int input_pixels,
int input_fps,
- int min_pixels_per_frame,
- AdaptationObserverInterface::AdaptReason reason) const {
+ int min_pixels_per_frame) const {
// Preconditions for being able to adapt down:
if (!has_input_video_)
return absl::nullopt;
@@ -860,22 +817,19 @@
switch (target.action) {
case AdaptationAction::kIncreaseResolution:
source_restrictor_->IncreaseResolutionTo(target.value);
- GetAdaptCounter().DecrementResolution(reason);
return;
case AdaptationAction::kDecreaseResolution:
source_restrictor_->DecreaseResolutionTo(target.value,
min_pixels_per_frame);
- GetAdaptCounter().IncrementResolution(reason);
return;
case AdaptationAction::kIncreaseFrameRate:
source_restrictor_->IncreaseFrameRateTo(target.value);
- GetAdaptCounter().DecrementFramerate(reason, target.value);
// TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
// GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
// in DecrementFramerate() makes it hard to predict whether this will be
// the last step. Remove the dependency on GetConstAdaptCounter().
if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
- GetConstAdaptCounter().FramerateCount() == 0 &&
+ source_restrictor_->adaptation_counters().fps_adaptations == 0 &&
target.value != std::numeric_limits<int>::max()) {
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
source_restrictor_->IncreaseFrameRateTo(
@@ -884,7 +838,6 @@
return;
case AdaptationAction::kDecreaseFrameRate:
source_restrictor_->DecreaseFrameRateTo(target.value);
- GetAdaptCounter().IncrementFramerate(reason);
return;
}
}
@@ -908,7 +861,7 @@
MaybeUpdateVideoSourceRestrictions();
// Stats and logging.
UpdateAdaptationStats(reason);
- RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString();
+ RTC_LOG(LS_INFO) << ActiveCountsToString();
}
ResourceListenerResponse
@@ -921,7 +874,7 @@
int min_pixels_per_frame = MinPixelsPerFrame();
// Should we adapt, if so to what target?
absl::optional<AdaptationTarget> target =
- GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame, reason);
+ GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame);
if (!target.has_value())
return ResourceListenerResponse::kNothing;
// Apply target.
@@ -933,7 +886,7 @@
MaybeUpdateVideoSourceRestrictions();
// Stats and logging.
UpdateAdaptationStats(reason);
- RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString();
+ RTC_LOG(INFO) << ActiveCountsToString();
// In BALANCED, if requested FPS is higher or close to input FPS to the target
// we tell the QualityScaler to increase its frequency.
if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
@@ -1035,9 +988,74 @@
encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
}
+void OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
+ const AdaptationCounters& adaptation_count,
+ AdaptationCounters* active_count,
+ AdaptationCounters* other_active) {
+ RTC_DCHECK(active_count);
+ RTC_DCHECK(other_active);
+ const int active_total = active_count->Total();
+ const int other_total = other_active->Total();
+ const AdaptationCounters prev_total = *active_count + *other_active;
+ const AdaptationCounters delta = adaptation_count - prev_total;
+
+ RTC_DCHECK_EQ(
+ std::abs(delta.resolution_adaptations) + std::abs(delta.fps_adaptations),
+ 1)
+ << "Adaptation took more than one step!";
+
+ if (delta.resolution_adaptations > 0) {
+ ++active_count->resolution_adaptations;
+ } else if (delta.resolution_adaptations < 0) {
+ if (active_count->resolution_adaptations == 0) {
+ RTC_DCHECK_GT(active_count->fps_adaptations, 0) << "No downgrades left";
+ RTC_DCHECK_GT(other_active->resolution_adaptations, 0)
+ << "No resolution adaptation to borrow from";
+ // Lend an fps adaptation to other and take one resolution adaptation.
+ --active_count->fps_adaptations;
+ ++other_active->fps_adaptations;
+ --other_active->resolution_adaptations;
+ } else {
+ --active_count->resolution_adaptations;
+ }
+ }
+ if (delta.fps_adaptations > 0) {
+ ++active_count->fps_adaptations;
+ } else if (delta.fps_adaptations < 0) {
+ if (active_count->fps_adaptations == 0) {
+ RTC_DCHECK_GT(active_count->resolution_adaptations, 0)
+ << "No downgrades left";
+ RTC_DCHECK_GT(other_active->fps_adaptations, 0)
+ << "No fps adaptation to borrow from";
+ // Lend a resolution adaptation to other and take one fps adaptation.
+ --active_count->resolution_adaptations;
+ ++other_active->resolution_adaptations;
+ --other_active->fps_adaptations;
+ } else {
+ --active_count->fps_adaptations;
+ }
+ }
+
+ RTC_DCHECK(*active_count + *other_active == adaptation_count);
+ RTC_DCHECK_EQ(other_active->Total(), other_total);
+ RTC_DCHECK_EQ(active_count->Total(), active_total + delta.Total());
+ RTC_DCHECK_GE(active_count->resolution_adaptations, 0);
+ RTC_DCHECK_GE(active_count->fps_adaptations, 0);
+ RTC_DCHECK_GE(other_active->resolution_adaptations, 0);
+ RTC_DCHECK_GE(other_active->fps_adaptations, 0);
+}
+
// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats(
AdaptationObserverInterface::AdaptReason reason) {
+ // Update active counts
+ AdaptationCounters& active_count = active_counts_[reason];
+ AdaptationCounters& other_active = active_counts_[(reason + 1) % 2];
+ const AdaptationCounters total_counts =
+ source_restrictor_->adaptation_counters();
+
+ OnAdaptationCountChanged(total_counts, &active_count, &other_active);
+
switch (reason) {
case AdaptationObserverInterface::AdaptReason::kCpu:
encoder_stats_observer_->OnAdaptationChanged(
@@ -1057,8 +1075,14 @@
VideoStreamEncoderObserver::AdaptationSteps
OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
AdaptationObserverInterface::AdaptReason reason) {
+ // TODO(https://crbug.com/webrtc/11392) Ideally this shuold be moved out of
+ // this class and into the encoder_stats_observer_.
+ const AdaptationCounters counters = active_counts_[reason];
+
VideoStreamEncoderObserver::AdaptationSteps counts =
- GetConstAdaptCounter().Counts(reason);
+ VideoStreamEncoderObserver::AdaptationSteps();
+ counts.num_resolution_reductions = counters.resolution_adaptations;
+ counts.num_framerate_reductions = counters.fps_adaptations;
switch (reason) {
case AdaptationObserverInterface::AdaptReason::kCpu:
if (!IsFramerateScalingEnabled(degradation_preference_))
@@ -1095,30 +1119,6 @@
: degradation_preference_;
}
-OveruseFrameDetectorResourceAdaptationModule::AdaptCounter&
-OveruseFrameDetectorResourceAdaptationModule::GetAdaptCounter() {
- return adapt_counters_[degradation_preference_];
-}
-
-void OveruseFrameDetectorResourceAdaptationModule::ClearAdaptCounters() {
- adapt_counters_.clear();
- adapt_counters_.insert(
- std::make_pair(DegradationPreference::DISABLED, AdaptCounter()));
- adapt_counters_.insert(std::make_pair(
- DegradationPreference::MAINTAIN_FRAMERATE, AdaptCounter()));
- adapt_counters_.insert(std::make_pair(
- DegradationPreference::MAINTAIN_RESOLUTION, AdaptCounter()));
- adapt_counters_.insert(
- std::make_pair(DegradationPreference::BALANCED, AdaptCounter()));
-}
-
-const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter&
-OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() const {
- auto it = adapt_counters_.find(degradation_preference_);
- RTC_DCHECK(it != adapt_counters_.cend());
- return it->second;
-}
-
bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
int pixels,
uint32_t bitrate_bps) const {
@@ -1158,15 +1158,36 @@
try_quality_rampup = true;
}
}
- if (try_quality_rampup &&
- GetConstAdaptCounter().ResolutionCount(
- AdaptationObserverInterface::AdaptReason::kQuality) > 0 &&
- GetConstAdaptCounter().TotalCount(
- AdaptationObserverInterface::AdaptReason::kCpu) == 0) {
+ // TODO(https://crbug.com/webrtc/11392): See if we can rely on the total
+ // counts or the stats, and not the active counts.
+ const AdaptationCounters& qp_counts =
+ std::get<AdaptationObserverInterface::kQuality>(active_counts_);
+ const AdaptationCounters& cpu_counts =
+ std::get<AdaptationObserverInterface::kCpu>(active_counts_);
+ if (try_quality_rampup && qp_counts.resolution_adaptations > 0 &&
+ cpu_counts.Total() == 0) {
RTC_LOG(LS_INFO) << "Reset quality limitations.";
ResetVideoSourceRestrictions();
quality_rampup_done_ = true;
}
}
+std::string OveruseFrameDetectorResourceAdaptationModule::ActiveCountsToString()
+ const {
+ rtc::StringBuilder ss;
+
+ ss << "Downgrade counts: fps: {";
+ for (size_t reason = 0; reason < active_counts_.size(); ++reason) {
+ ss << (reason ? " cpu" : "quality") << ":";
+ ss << active_counts_[reason].fps_adaptations;
+ }
+ ss << "}, resolution {";
+ for (size_t reason = 0; reason < active_counts_.size(); ++reason) {
+ ss << (reason ? " cpu" : "quality") << ":";
+ ss << active_counts_[reason].resolution_adaptations;
+ }
+ ss << "}";
+
+ return ss.Release();
+}
} // namespace webrtc
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
index 6e846d7..30f0aa3 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.h
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -30,6 +30,7 @@
#include "rtc_base/experiments/balanced_degradation_settings.h"
#include "rtc_base/experiments/quality_rampup_experiment.h"
#include "rtc_base/experiments/quality_scaler_settings.h"
+#include "rtc_base/strings/string_builder.h"
#include "system_wrappers/include/clock.h"
#include "video/encode_usage_resource.h"
#include "video/overuse_frame_detector.h"
@@ -37,6 +38,26 @@
namespace webrtc {
+// Counts the number of adaptations have resulted due to resource overuse.
+// Today we can adapt resolution and fps.
+struct AdaptationCounters {
+ AdaptationCounters() : resolution_adaptations(0), fps_adaptations(0) {}
+ AdaptationCounters(int resolution_adaptations, int fps_adaptations)
+ : resolution_adaptations(resolution_adaptations),
+ fps_adaptations(fps_adaptations) {}
+
+ int Total() const { return fps_adaptations + resolution_adaptations; }
+
+ bool operator==(const AdaptationCounters& rhs) const;
+ bool operator!=(const AdaptationCounters& rhs) const;
+
+ AdaptationCounters operator+(const AdaptationCounters& other) const;
+ AdaptationCounters operator-(const AdaptationCounters& other) const;
+
+ int resolution_adaptations;
+ int fps_adaptations;
+};
+
class VideoStreamEncoder;
// This class is used by the VideoStreamEncoder and is responsible for adapting
@@ -114,9 +135,20 @@
ResourceListenerResponse OnResourceUsageStateMeasured(
const Resource& resource) override;
+ // For reasons of adaptation and statistics, we not only count the total
+ // number of adaptations, but we also count the number of adaptations per
+ // reason.
+ // This method takes the new total number of adaptations and allocates that to
+ // the "active" count - number of adaptations for the current reason.
+ // The "other" count is the number of adaptations for the other reason.
+ // This must be called for each adaptation step made.
+ static void OnAdaptationCountChanged(
+ const AdaptationCounters& adaptation_count,
+ AdaptationCounters* active_count,
+ AdaptationCounters* other_active);
+
private:
class VideoSourceRestrictor;
- class AdaptCounter;
class InitialFrameDropper;
enum class State { kStopped, kStarted };
@@ -160,8 +192,7 @@
absl::optional<AdaptationTarget> GetAdaptDownTarget(
int input_pixels,
int input_fps,
- int min_pixels_per_frame,
- AdaptationObserverInterface::AdaptReason reason) const;
+ int min_pixels_per_frame) const;
// Applies the |target| to |source_restrictor_|.
void ApplyAdaptationTarget(const AdaptationTarget& target,
int min_pixels_per_frame,
@@ -179,8 +210,6 @@
int MinPixelsPerFrame() const;
VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
AdaptationObserverInterface::AdaptReason reason);
- void ClearAdaptCounters();
- const AdaptCounter& GetConstAdaptCounter() const;
// Makes |video_source_restrictions_| up-to-date and informs the
// |adaptation_listener_| if restrictions are changed, allowing the listener
@@ -196,7 +225,6 @@
void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason);
DegradationPreference EffectiveDegradationPreference() const;
- AdaptCounter& GetAdaptCounter();
bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const;
// Checks to see if we should execute the quality rampup experiment. The
@@ -207,6 +235,8 @@
void MaybePerformQualityRampupExperiment();
void ResetVideoSourceRestrictions();
+ std::string ActiveCountsToString() const;
+
ResourceAdaptationModuleListener* const adaptation_listener_;
Clock* clock_;
State state_;
@@ -215,12 +245,6 @@
VideoSourceRestrictions video_source_restrictions_;
bool has_input_video_;
DegradationPreference degradation_preference_;
- // Counters used for deciding if the video resolution or framerate is
- // currently restricted, and if so, why, on a per degradation preference
- // basis.
- // TODO(sprang): Replace this with a state holding a relative overuse measure
- // instead, that can be translated into suitable down-scale or fps limit.
- std::map<const DegradationPreference, AdaptCounter> adapt_counters_;
const BalancedDegradationSettings balanced_settings_;
// Stores a snapshot of the last adaptation request triggered by an AdaptUp
// or AdaptDown signal.
@@ -253,6 +277,15 @@
const AdaptationObserverInterface::AdaptReason reason;
};
std::vector<ResourceAndReason> resources_;
+ // One AdaptationCounter for each reason, tracking the number of times we have
+ // adapted for each reason. The sum of active_counts_ MUST always equal the
+ // total adaptation provided by the VideoSourceRestrictions.
+ // TODO(https://crbug.com/webrtc/11392): Move all active count logic to
+ // encoder_stats_observer_; Counters used for deciding if the video resolution
+ // or framerate is currently restricted, and if so, why, on a per degradation
+ // preference basis.
+ std::array<AdaptationCounters, AdaptationObserverInterface::kScaleReasonSize>
+ active_counts_;
};
} // namespace webrtc
diff --git a/video/overuse_frame_detector_resource_adaptation_unittest.cc b/video/overuse_frame_detector_resource_adaptation_unittest.cc
new file mode 100644
index 0000000..428618b
--- /dev/null
+++ b/video/overuse_frame_detector_resource_adaptation_unittest.cc
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/overuse_frame_detector_resource_adaptation_module.h"
+
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+TEST(AdaptationCountersTest, Addition) {
+ AdaptationCounters a;
+ AdaptationCounters b(1, 2);
+ AdaptationCounters total = a + b;
+ EXPECT_EQ(1, total.resolution_adaptations);
+ EXPECT_EQ(2, total.fps_adaptations);
+}
+
+TEST(AdaptationCountersTest, Subtraction) {
+ AdaptationCounters a(0, 1);
+ AdaptationCounters b(2, 1);
+ AdaptationCounters diff = a - b;
+ EXPECT_EQ(-2, diff.resolution_adaptations);
+ EXPECT_EQ(0, diff.fps_adaptations);
+}
+
+TEST(AdaptationCountersTest, Equality) {
+ AdaptationCounters a(1, 2);
+ AdaptationCounters b(2, 1);
+ EXPECT_EQ(a, a);
+ EXPECT_NE(a, b);
+}
+
+TEST(AdaptationCountersTest, SelfAdditionSubtraction) {
+ AdaptationCounters a(1, 0);
+ AdaptationCounters b(0, 1);
+
+ EXPECT_EQ(a, a + b - b);
+ EXPECT_EQ(a, b + a - b);
+ EXPECT_EQ(a, a - b + b);
+ EXPECT_EQ(a, b - b + a);
+}
+
+TEST(OveruseFrameDetectorResourceAdaptationModuleTest,
+ FirstAdaptationDown_Fps) {
+ AdaptationCounters cpu;
+ AdaptationCounters qp;
+ AdaptationCounters total(0, 1);
+
+ OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
+ total, &cpu, &qp);
+ AdaptationCounters expected_cpu(0, 1);
+ AdaptationCounters expected_qp;
+ EXPECT_EQ(expected_cpu, cpu);
+ EXPECT_EQ(expected_qp, qp);
+}
+
+TEST(OveruseFrameDetectorResourceAdaptationModuleTest,
+ FirstAdaptationDown_Resolution) {
+ AdaptationCounters cpu;
+ AdaptationCounters qp;
+ AdaptationCounters total(1, 0);
+
+ OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
+ total, &cpu, &qp);
+ AdaptationCounters expected_cpu(1, 0);
+ AdaptationCounters expected_qp;
+ EXPECT_EQ(expected_cpu, cpu);
+ EXPECT_EQ(expected_qp, qp);
+}
+
+TEST(OveruseFrameDetectorResourceAdaptationModuleTest, LastAdaptUp_Fps) {
+ AdaptationCounters cpu(0, 1);
+ AdaptationCounters qp;
+ AdaptationCounters total;
+
+ OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
+ total, &cpu, &qp);
+ AdaptationCounters expected_cpu;
+ AdaptationCounters expected_qp;
+ EXPECT_EQ(expected_cpu, cpu);
+ EXPECT_EQ(expected_qp, qp);
+}
+
+TEST(OveruseFrameDetectorResourceAdaptationModuleTest, LastAdaptUp_Resolution) {
+ AdaptationCounters cpu(1, 0);
+ AdaptationCounters qp;
+ AdaptationCounters total;
+
+ OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
+ total, &cpu, &qp);
+ AdaptationCounters expected_cpu;
+ AdaptationCounters expected_qp;
+ EXPECT_EQ(expected_cpu, cpu);
+ EXPECT_EQ(expected_qp, qp);
+}
+
+TEST(OveruseFrameDetectorResourceAdaptationModuleTest,
+ AdaptUpWithBorrow_Resolution) {
+ AdaptationCounters cpu(0, 1);
+ AdaptationCounters qp(1, 0);
+ AdaptationCounters total(0, 1);
+
+ // CPU adaptation for resolution, but no
+ // resolution adaptation left from CPU.
+ // We then borrow the resolution
+ // adaptation from qp, and give qp the
+ // fps adaptation from CPU.
+ OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
+ total, &cpu, &qp);
+
+ AdaptationCounters expected_cpu(0, 0);
+ AdaptationCounters expected_qp(0, 1);
+ EXPECT_EQ(expected_cpu, cpu);
+ EXPECT_EQ(expected_qp, qp);
+}
+
+TEST(OveruseFrameDetectorResourceAdaptationModuleTest, AdaptUpWithBorrow_Fps) {
+ AdaptationCounters cpu(1, 0);
+ AdaptationCounters qp(0, 1);
+ AdaptationCounters total(1, 0);
+
+ // CPU adaptation for fps, but no
+ // fps adaptation left from CPU. We
+ // then borrow the fps adaptation
+ // from qp, and give qp the
+ // resolution adaptation from CPU.
+ OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
+ total, &cpu, &qp);
+
+ AdaptationCounters expected_cpu(0, 0);
+ AdaptationCounters expected_qp(1, 0);
+ EXPECT_EQ(expected_cpu, cpu);
+ EXPECT_EQ(expected_qp, qp);
+}
+
+} // namespace webrtc
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index e48ccd8..9d6925e 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -2236,6 +2236,73 @@
}
TEST_F(VideoStreamEncoderTest,
+ StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
+ video_stream_encoder_->OnBitrateUpdated(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ int sequence = 1;
+
+ // Enable BALANCED preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU overuse, should now adapt down.
+ video_stream_encoder_->TriggerCpuOveruse();
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Set new degradation preference should clear restrictions since we changed
+ // from BALANCED.
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Force an input frame rate to be available, or the adaptation call won't
+ // know what framerate to adapt from.
+ VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
+ mock_stats.input_frame_rate = 30;
+ stats_proxy_->SetMockStats(mock_stats);
+ video_stream_encoder_->TriggerCpuOveruse();
+ stats_proxy_->ResetMockStats();
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+
+ // We have now adapted once.
+ stats = stats_proxy_->GetStats();
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+
+ // Back to BALANCED, should clear the restrictions again.
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
video_stream_encoder_->OnBitrateUpdated(
DataRate::BitsPerSec(kTargetBitrateBps),
