Refactor OverUseFrameDetector to use the timestamps attached to EncodedImage.
Bug: webrtc:8504
Change-Id: I3f99c3c1e528f59b45724921a91f65b485f5b820
Reviewed-on: https://webrtc-review.googlesource.com/23720
Commit-Queue: Niels Moller <nisse@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20979}diff --git a/video/overuse_frame_detector.cc b/video/overuse_frame_detector.cc
index 7b64c10..8519ce7 100644
--- a/video/overuse_frame_detector.cc
+++ b/video/overuse_frame_detector.cc
@@ -23,7 +23,6 @@
#include "common_video/include/frame_callback.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
-#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/timeutils.h"
#include "system_wrappers/include/field_trial.h"
@@ -48,31 +47,15 @@
// Max number of overuses detected before always applying the rampup delay.
const int kMaxOverusesBeforeApplyRampupDelay = 4;
-// The maximum exponent to use in VCMExpFilter.
-const float kMaxExp = 7.0f;
-// Default value used before first reconfiguration.
-const int kDefaultFrameRate = 30;
-// Default sample diff, default frame rate.
-const float kDefaultSampleDiffMs = 1000.0f / kDefaultFrameRate;
-// A factor applied to the sample diff on OnTargetFramerateUpdated to determine
-// a max limit for the sample diff. For instance, with a framerate of 30fps,
-// the sample diff is capped to (1000 / 30) * 1.35 = 45ms. This prevents
-// triggering too soon if there are individual very large outliers.
-const float kMaxSampleDiffMarginFactor = 1.35f;
-// Minimum framerate allowed for usage calculation. This prevents crazy long
-// encode times from being accepted if the frame rate happens to be low.
-const int kMinFramerate = 7;
-const int kMaxFramerate = 30;
-
const auto kScaleReasonCpu = AdaptationObserverInterface::AdaptReason::kCpu;
} // namespace
CpuOveruseOptions::CpuOveruseOptions()
: high_encode_usage_threshold_percent(85),
frame_timeout_interval_ms(1500),
- min_frame_samples(120),
min_process_count(3),
- high_threshold_consecutive_count(2) {
+ high_threshold_consecutive_count(2),
+ filter_time_ms(5000) {
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
// This is proof-of-concept code for letting the physical core count affect
// the interval into which we attempt to scale. For now, the code is Mac OS
@@ -123,72 +106,43 @@
class OveruseFrameDetector::SendProcessingUsage {
public:
explicit SendProcessingUsage(const CpuOveruseOptions& options)
- : kWeightFactorFrameDiff(0.998f),
- kWeightFactorProcessing(0.995f),
- kInitialSampleDiffMs(40.0f),
- count_(0),
- options_(options),
- max_sample_diff_ms_(kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor),
- filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)),
- filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) {
+ : options_(options) {
Reset();
}
virtual ~SendProcessingUsage() {}
void Reset() {
- count_ = 0;
- max_sample_diff_ms_ = kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor;
- filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff);
- filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs);
- filtered_processing_ms_->Reset(kWeightFactorProcessing);
- filtered_processing_ms_->Apply(1.0f, InitialProcessingMs());
+ // Start in between the underuse and overuse threshold.
+ load_estimate_ = (options_.low_encode_usage_threshold_percent +
+ options_.high_encode_usage_threshold_percent) /
+ 200.0;
}
- void SetMaxSampleDiffMs(float diff_ms) { max_sample_diff_ms_ = diff_ms; }
+ void AddSample(double encode_time, double diff_time) {
+ RTC_CHECK_GE(diff_time, 0.0);
- void AddCaptureSample(float sample_ms) {
- float exp = sample_ms / kDefaultSampleDiffMs;
- exp = std::min(exp, kMaxExp);
- filtered_frame_diff_ms_->Apply(exp, sample_ms);
- }
-
- void AddSample(float processing_ms, int64_t diff_last_sample_ms) {
- ++count_;
- float exp = diff_last_sample_ms / kDefaultSampleDiffMs;
- exp = std::min(exp, kMaxExp);
- filtered_processing_ms_->Apply(exp, processing_ms);
- }
-
- virtual int Value() {
- if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) {
- return static_cast<int>(InitialUsageInPercent() + 0.5f);
+ // Use the filter update
+ //
+ // load <-- x/d (1-exp (-d/T)) + exp (-d/T) load
+ //
+ // where we must take care for small d, using the proper limit
+ // (1 - exp(-d/tau)) / d = 1/tau - d/2tau^2 + O(d^2)
+ double tau = (1e-3 * options_.filter_time_ms);
+ double e = diff_time / tau;
+ double c;
+ if (e < 0.0001) {
+ c = (1 - e / 2) / tau;
+ } else {
+ c = -expm1(-e) / diff_time;
}
- float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f);
- frame_diff_ms = std::min(frame_diff_ms, max_sample_diff_ms_);
- float encode_usage_percent =
- 100.0f * filtered_processing_ms_->filtered() / frame_diff_ms;
- return static_cast<int>(encode_usage_percent + 0.5);
+ load_estimate_ = c * encode_time + exp(-e) * load_estimate_;
}
+ virtual int Value() { return static_cast<int>(100.0 * load_estimate_ + 0.5); }
+
private:
- float InitialUsageInPercent() const {
- // Start in between the underuse and overuse threshold.
- return (options_.low_encode_usage_threshold_percent +
- options_.high_encode_usage_threshold_percent) / 2.0f;
- }
-
- float InitialProcessingMs() const {
- return InitialUsageInPercent() * kInitialSampleDiffMs / 100;
- }
-
- const float kWeightFactorFrameDiff;
- const float kWeightFactorProcessing;
- const float kInitialSampleDiffMs;
- uint64_t count_;
const CpuOveruseOptions options_;
- float max_sample_diff_ms_;
- std::unique_ptr<rtc::ExpFilter> filtered_processing_ms_;
- std::unique_ptr<rtc::ExpFilter> filtered_frame_diff_ms_;
+ double load_estimate_;
};
// Class used for manual testing of overuse, enabled via field trial flag.
@@ -218,6 +172,7 @@
int64_t now_ms = rtc::TimeMillis();
if (last_toggling_ms_ == -1) {
last_toggling_ms_ = now_ms;
+
} else {
switch (state_) {
case State::kNormal:
@@ -255,7 +210,6 @@
overried_usage_value.emplace(5);
break;
}
-
return overried_usage_value.value_or(SendProcessingUsage::Value());
}
@@ -348,7 +302,6 @@
last_capture_time_us_(-1),
last_processed_capture_time_us_(-1),
num_pixels_(0),
- max_framerate_(kDefaultFrameRate),
last_overuse_time_ms_(-1),
checks_above_threshold_(0),
num_overuse_detections_(0),
@@ -399,93 +352,41 @@
options_.frame_timeout_interval_ms * rtc::kNumMicrosecsPerMillisec;
}
-void OveruseFrameDetector::ResetAll(int num_pixels) {
- // Reset state, as a result resolution being changed. Do not however change
- // the current frame rate back to the default.
+void OveruseFrameDetector::ResetAll() {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
- num_pixels_ = num_pixels;
usage_->Reset();
- frame_timing_.clear();
last_capture_time_us_ = -1;
last_processed_capture_time_us_ = -1;
num_process_times_ = 0;
metrics_ = rtc::Optional<CpuOveruseMetrics>();
- OnTargetFramerateUpdated(max_framerate_);
}
-void OveruseFrameDetector::OnTargetFramerateUpdated(int framerate_fps) {
- RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
- RTC_DCHECK_GE(framerate_fps, 0);
- max_framerate_ = std::min(kMaxFramerate, framerate_fps);
- usage_->SetMaxSampleDiffMs((1000 / std::max(kMinFramerate, max_framerate_)) *
- kMaxSampleDiffMarginFactor);
-}
-
-void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame,
- int64_t time_when_first_seen_us) {
+void OveruseFrameDetector::FrameCaptured(int width, int height) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
- if (FrameSizeChanged(frame.width() * frame.height()) ||
- FrameTimeoutDetected(time_when_first_seen_us)) {
- ResetAll(frame.width() * frame.height());
+ if (FrameSizeChanged(width * height)) {
+ ResetAll();
+ num_pixels_ = width * height;
}
-
- if (last_capture_time_us_ != -1)
- usage_->AddCaptureSample(
- 1e-3 * (time_when_first_seen_us - last_capture_time_us_));
-
- last_capture_time_us_ = time_when_first_seen_us;
-
- frame_timing_.push_back(FrameTiming(frame.timestamp_us(), frame.timestamp(),
- time_when_first_seen_us));
}
-void OveruseFrameDetector::FrameSent(uint32_t timestamp,
- int64_t time_sent_in_us) {
+void OveruseFrameDetector::FrameEncoded(int64_t capture_time_us,
+ int64_t encode_duration_us) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
- // Delay before reporting actual encoding time, used to have the ability to
- // detect total encoding time when encoding more than one layer. Encoding is
- // here assumed to finish within a second (or that we get enough long-time
- // samples before one second to trigger an overuse even when this is not the
- // case).
- static const int64_t kEncodingTimeMeasureWindowMs = 1000;
- for (auto& it : frame_timing_) {
- if (it.timestamp == timestamp) {
- it.last_send_us = time_sent_in_us;
- break;
- }
+ if (FrameTimeoutDetected(capture_time_us)) {
+ ResetAll();
+ } else if (last_capture_time_us_ != -1) {
+ usage_->AddSample(1e-6 * encode_duration_us,
+ 1e-6 * (capture_time_us - last_capture_time_us_));
}
- // TODO(pbos): Handle the case/log errors when not finding the corresponding
- // frame (either very slow encoding or incorrect wrong timestamps returned
- // from the encoder).
- // This is currently the case for all frames on ChromeOS, so logging them
- // would be spammy, and triggering overuse would be wrong.
- // https://crbug.com/350106
- while (!frame_timing_.empty()) {
- FrameTiming timing = frame_timing_.front();
- if (time_sent_in_us - timing.capture_us <
- kEncodingTimeMeasureWindowMs * rtc::kNumMicrosecsPerMillisec) {
- break;
- }
- if (timing.last_send_us != -1) {
- int encode_duration_us =
- static_cast<int>(timing.last_send_us - timing.capture_us);
- if (encoder_timing_) {
- // TODO(nisse): Update encoder_timing_ to also use us units.
- encoder_timing_->OnEncodeTiming(timing.capture_time_us /
- rtc::kNumMicrosecsPerMillisec,
- encode_duration_us /
- rtc::kNumMicrosecsPerMillisec);
- }
- if (last_processed_capture_time_us_ != -1) {
- int64_t diff_us = timing.capture_us - last_processed_capture_time_us_;
- usage_->AddSample(1e-3 * encode_duration_us, 1e-3 * diff_us);
- }
- last_processed_capture_time_us_ = timing.capture_us;
- EncodedFrameTimeMeasured(encode_duration_us /
- rtc::kNumMicrosecsPerMillisec);
- }
- frame_timing_.pop_front();
+ last_capture_time_us_ = capture_time_us;
+ EncodedFrameTimeMeasured(encode_duration_us / rtc::kNumMicrosecsPerMillisec);
+
+ if (encoder_timing_) {
+ // TODO(nisse): Update encoder_timing_ to also use us units.
+ encoder_timing_->OnEncodeTiming(
+ capture_time_us / rtc::kNumMicrosecsPerMillisec,
+ encode_duration_us / rtc::kNumMicrosecsPerMillisec);
}
}
diff --git a/video/overuse_frame_detector.h b/video/overuse_frame_detector.h
index 3cc9262..2ec5230 100644
--- a/video/overuse_frame_detector.h
+++ b/video/overuse_frame_detector.h
@@ -17,7 +17,6 @@
#include "api/optional.h"
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/constructormagic.h"
-#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/sequenced_task_checker.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/thread_annotations.h"
@@ -35,12 +34,12 @@
// General settings.
int frame_timeout_interval_ms; // The maximum allowed interval between two
// frames before resetting estimations.
- int min_frame_samples; // The minimum number of frames required.
int min_process_count; // The number of initial process times required before
// triggering an overuse/underuse.
int high_threshold_consecutive_count; // The number of consecutive checks
// above the high threshold before
// triggering an overuse.
+ int filter_time_ms; // Time constant for averaging
};
struct CpuOveruseMetrics {
@@ -77,18 +76,11 @@
// StartCheckForOveruse has been called.
void StopCheckForOveruse();
- // Defines the current maximum framerate targeted by the capturer. This is
- // used to make sure the encode usage percent doesn't drop unduly if the
- // capturer has quiet periods (for instance caused by screen capturers with
- // variable capture rate depending on content updates), otherwise we might
- // experience adaptation toggling.
- virtual void OnTargetFramerateUpdated(int framerate_fps);
-
// Called for each captured frame.
- void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us);
+ void FrameCaptured(int width, int height);
- // Called for each sent frame.
- void FrameSent(uint32_t timestamp, int64_t time_sent_in_us);
+ // Called for each encoded frame.
+ void FrameEncoded(int64_t capture_time_us, int64_t encode_duration_us);
protected:
void CheckForOveruse(); // Protected for test purposes.
@@ -97,17 +89,6 @@
class OverdoseInjector;
class SendProcessingUsage;
class CheckOveruseTask;
- struct FrameTiming {
- FrameTiming(int64_t capture_time_us, uint32_t timestamp, int64_t now)
- : capture_time_us(capture_time_us),
- timestamp(timestamp),
- capture_us(now),
- last_send_us(-1) {}
- int64_t capture_time_us;
- uint32_t timestamp;
- int64_t capture_us;
- int64_t last_send_us;
- };
void EncodedFrameTimeMeasured(int encode_duration_ms);
bool IsOverusing(const CpuOveruseMetrics& metrics);
@@ -116,7 +97,7 @@
bool FrameTimeoutDetected(int64_t now) const;
bool FrameSizeChanged(int num_pixels) const;
- void ResetAll(int num_pixels);
+ void ResetAll();
static std::unique_ptr<SendProcessingUsage> CreateSendProcessingUsage(
const CpuOveruseOptions& options);
@@ -142,7 +123,6 @@
// Number of pixels of last captured frame.
int num_pixels_ RTC_GUARDED_BY(task_checker_);
- int max_framerate_ RTC_GUARDED_BY(task_checker_);
int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_);
int checks_above_threshold_ RTC_GUARDED_BY(task_checker_);
int num_overuse_detections_ RTC_GUARDED_BY(task_checker_);
@@ -154,7 +134,6 @@
// allocs)?
const std::unique_ptr<SendProcessingUsage> usage_
RTC_GUARDED_BY(task_checker_);
- std::list<FrameTiming> frame_timing_ RTC_GUARDED_BY(task_checker_);
RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector);
};
diff --git a/video/overuse_frame_detector_unittest.cc b/video/overuse_frame_detector_unittest.cc
index 0f3dd86..440d86b 100644
--- a/video/overuse_frame_detector_unittest.cc
+++ b/video/overuse_frame_detector_unittest.cc
@@ -15,6 +15,8 @@
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/event.h"
#include "rtc_base/fakeclock.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/random.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "video/overuse_frame_detector.h"
@@ -22,6 +24,8 @@
namespace webrtc {
using ::testing::InvokeWithoutArgs;
+using ::testing::AtLeast;
+using ::testing::_;
namespace {
const int kWidth = 640;
@@ -97,27 +101,20 @@
int width,
int height,
int delay_us) {
- VideoFrame frame(I420Buffer::Create(width, height),
- webrtc::kVideoRotation_0, 0);
- uint32_t timestamp = 0;
while (num_frames-- > 0) {
- frame.set_timestamp(timestamp);
- overuse_detector_->FrameCaptured(frame, rtc::TimeMicros());
- clock_.AdvanceTimeMicros(delay_us);
- overuse_detector_->FrameSent(timestamp, rtc::TimeMicros());
- clock_.AdvanceTimeMicros(interval_us - delay_us);
- timestamp += interval_us * 90 / 1000;
+ overuse_detector_->FrameCaptured(width, height);
+ overuse_detector_->FrameEncoded(rtc::TimeMicros(), delay_us);
+ clock_.AdvanceTimeMicros(interval_us);
}
}
void ForceUpdate(int width, int height) {
- // Insert one frame, wait a second and then put in another to force update
- // the usage. From the tests where these are used, adding another sample
- // doesn't affect the expected outcome (this is mainly to check initial
- // values and whether the overuse detector has been reset or not).
- InsertAndSendFramesWithInterval(2, rtc::kNumMicrosecsPerSec,
- width, height, kFrameIntervalUs);
+ // This is mainly to check initial values and whether the overuse
+ // detector has been reset or not.
+ InsertAndSendFramesWithInterval(1, rtc::kNumMicrosecsPerSec, width, height,
+ kFrameIntervalUs);
}
+
void TriggerOveruse(int num_times) {
const int kDelayUs = 32 * rtc::kNumMicrosecsPerMillisec;
for (int i = 0; i < num_times; ++i) {
@@ -269,76 +266,11 @@
EXPECT_EQ(InitialUsage(), UsagePercent());
}
-TEST_F(OveruseFrameDetectorTest, MinFrameSamplesBeforeUpdating) {
- options_.min_frame_samples = 40;
- ReinitializeOveruseDetector();
- InsertAndSendFramesWithInterval(
- 40, kFrameIntervalUs, kWidth, kHeight, kProcessTimeUs);
- EXPECT_EQ(InitialUsage(), UsagePercent());
- // Pass time far enough to digest all previous samples.
- clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec);
- InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight,
- kProcessTimeUs);
- // The last sample has not been processed here.
- EXPECT_EQ(InitialUsage(), UsagePercent());
-
- // Pass time far enough to digest all previous samples, 41 in total.
- clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec);
- InsertAndSendFramesWithInterval(
- 1, kFrameIntervalUs, kWidth, kHeight, kProcessTimeUs);
- EXPECT_NE(InitialUsage(), UsagePercent());
-}
-
TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) {
ForceUpdate(kWidth, kHeight);
EXPECT_EQ(InitialUsage(), UsagePercent());
}
-TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) {
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_))
- .Times(testing::AtLeast(1));
- static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec;
- static const size_t kNumFramesEncodingDelay = 3;
- VideoFrame frame(I420Buffer::Create(kWidth, kHeight),
- webrtc::kVideoRotation_0, 0);
- for (size_t i = 0; i < 1000; ++i) {
- // Unique timestamps.
- frame.set_timestamp(static_cast<uint32_t>(i));
- overuse_detector_->FrameCaptured(frame, rtc::TimeMicros());
- clock_.AdvanceTimeMicros(kIntervalUs);
- if (i > kNumFramesEncodingDelay) {
- overuse_detector_->FrameSent(
- static_cast<uint32_t>(i - kNumFramesEncodingDelay),
- rtc::TimeMicros());
- }
- overuse_detector_->CheckForOveruse();
- }
-}
-
-TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) {
- // >85% encoding time should trigger overuse.
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_))
- .Times(testing::AtLeast(1));
- static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec;
- static const int kDelayUs = 30 * rtc::kNumMicrosecsPerMillisec;
- VideoFrame frame(I420Buffer::Create(kWidth, kHeight),
- webrtc::kVideoRotation_0, 0);
- uint32_t timestamp = 0;
- for (size_t i = 0; i < 1000; ++i) {
- frame.set_timestamp(timestamp);
- overuse_detector_->FrameCaptured(frame, rtc::TimeMicros());
- // Encode and send first parts almost instantly.
- clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec);
- overuse_detector_->FrameSent(timestamp, rtc::TimeMicros());
- // Encode heavier part, resulting in >85% usage total.
- clock_.AdvanceTimeMicros(kDelayUs - rtc::kNumMicrosecsPerMillisec);
- overuse_detector_->FrameSent(timestamp, rtc::TimeMicros());
- clock_.AdvanceTimeMicros(kIntervalUs - kDelayUs);
- timestamp += kIntervalUs * 90 / 1000;
- overuse_detector_->CheckForOveruse();
- }
-}
-
TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) {
rtc::TaskQueue queue("OveruseFrameDetectorTestQueue");
@@ -369,115 +301,59 @@
EXPECT_TRUE(event.Wait(10000));
}
-TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) {
- const int kCapturerMaxFrameRate = 30;
- const int kEncodeMaxFrameRate = 20; // Maximum fps the encoder can sustain.
+// Models screencast, with irregular arrival of frames which are heavy
+// to encode.
+TEST_F(OveruseFrameDetectorTest, NoOveruseForLargeRandomFrameInterval) {
+ EXPECT_CALL(*(observer_.get()), AdaptDown(_)).Times(0);
+ EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(AtLeast(1));
- // Trigger overuse.
- int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kCapturerMaxFrameRate;
- // Processing time just below over use limit given kEncodeMaxFrameRate.
- int64_t processing_time_us =
- (98 * OveruseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100;
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
- for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
- InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
- processing_time_us);
+ const int kNumFrames = 500;
+ const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec;
+
+ const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec;
+ const int kMaxIntervalUs = 1000 * rtc::kNumMicrosecsPerMillisec;
+
+ webrtc::Random random(17);
+
+ for (int i = 0; i < kNumFrames; i++) {
+ int interval_us = random.Rand(kMinIntervalUs, kMaxIntervalUs);
+ overuse_detector_->FrameCaptured(kWidth, kHeight);
+ overuse_detector_->FrameEncoded(rtc::TimeMicros(), kEncodeTimeUs);
+
overuse_detector_->CheckForOveruse();
+ clock_.AdvanceTimeMicros(interval_us);
}
-
- // Simulate frame rate reduction and normal usage.
- frame_interval_us = rtc::kNumMicrosecsPerSec / kEncodeMaxFrameRate;
- overuse_detector_->OnTargetFramerateUpdated(kEncodeMaxFrameRate);
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0);
- for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
- InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
- processing_time_us);
- overuse_detector_->CheckForOveruse();
- }
-
- // Reduce processing time to trigger underuse.
- processing_time_us =
- (98 * UnderuseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100;
- EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(1);
- InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
- processing_time_us);
- overuse_detector_->CheckForOveruse();
+ // Average usage 19%. Check that estimate is in the right ball park.
+ EXPECT_NEAR(UsagePercent(), 20, 10);
}
-TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) {
- const int kMinFrameRate = 7; // Minimum fps allowed by current detector impl.
- overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate);
+// Models screencast, with irregular arrival of frames, often
+// exceeding the timeout interval.
+TEST_F(OveruseFrameDetectorTest, NoOveruseForRandomFrameIntervalWithReset) {
+ EXPECT_CALL(*(observer_.get()), AdaptDown(_)).Times(0);
+ EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(AtLeast(1));
- // Normal usage just at the limit.
- int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMinFrameRate;
- // Processing time just below over use limit given kEncodeMaxFrameRate.
- int64_t processing_time_us =
- (98 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100;
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0);
- for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
- InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
- processing_time_us);
+ const int kNumFrames = 500;
+ const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec;
+
+ const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec;
+ const int kMaxIntervalUs = 3000 * rtc::kNumMicrosecsPerMillisec;
+
+ webrtc::Random random(17);
+
+ for (int i = 0; i < kNumFrames; i++) {
+ int interval_us = random.Rand(kMinIntervalUs, kMaxIntervalUs);
+ overuse_detector_->FrameCaptured(kWidth, kHeight);
+ overuse_detector_->FrameEncoded(rtc::TimeMicros(), kEncodeTimeUs);
+
overuse_detector_->CheckForOveruse();
+ clock_.AdvanceTimeMicros(interval_us);
}
-
- // Over the limit to overuse.
- processing_time_us =
- (102 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100;
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
- for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
- InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
- processing_time_us);
- overuse_detector_->CheckForOveruse();
- }
-
- // Reduce input frame rate. Should still trigger overuse.
- overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate - 1);
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
- for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
- InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
- processing_time_us);
- overuse_detector_->CheckForOveruse();
- }
-}
-
-TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) {
- const int kMaxFrameRate = 20;
- overuse_detector_->OnTargetFramerateUpdated(kMaxFrameRate);
- int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMaxFrameRate;
- // Maximum frame interval allowed is 35% above ideal.
- int64_t max_frame_interval_us = (135 * frame_interval_us) / 100;
- // Maximum processing time, without triggering overuse, allowed with the above
- // frame interval.
- int64_t max_processing_time_us =
- (max_frame_interval_us * options_.high_encode_usage_threshold_percent) /
- 100;
-
- // Processing time just below overuse limit given kMaxFrameRate.
- int64_t processing_time_us = (98 * max_processing_time_us) / 100;
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0);
- for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
- InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth,
- kHeight, processing_time_us);
- overuse_detector_->CheckForOveruse();
- }
-
- // Go above limit, trigger overuse.
- processing_time_us = (102 * max_processing_time_us) / 100;
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
- for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
- InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth,
- kHeight, processing_time_us);
- overuse_detector_->CheckForOveruse();
- }
-
- // Increase frame interval, should still trigger overuse.
- max_frame_interval_us *= 2;
- EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
- for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
- InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth,
- kHeight, processing_time_us);
- overuse_detector_->CheckForOveruse();
- }
+ // Average usage 6.6%, but since the frame_timeout_interval_ms is
+ // only 1500 ms, we often reset the estimate to the initial value.
+ // Check that estimate is in the right ball park.
+ EXPECT_GE(UsagePercent(), 1);
+ EXPECT_LE(UsagePercent(), InitialUsage() + 5);
}
} // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 77945dc..1bca22c 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -508,12 +508,6 @@
bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_);
initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop;
ConfigureQualityScaler();
- if (!IsFramerateScalingEnabled(degradation_preference) &&
- max_framerate_ != -1) {
- // If frame rate scaling is no longer allowed, remove any potential
- // allowance for longer frame intervals.
- overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
- }
});
}
@@ -628,15 +622,6 @@
sink_->OnEncoderConfigurationChanged(
std::move(streams), encoder_config_.min_transmit_bitrate_bps);
- // Get the current target framerate, ie the maximum framerate as specified by
- // the current codec configuration, or any limit imposed by cpu adaption in
- // maintain-resolution or balanced mode. This is used to make sure overuse
- // detection doesn't needlessly trigger in low and/or variable framerate
- // scenarios.
- int target_framerate = std::min(
- max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps);
- overuse_detector_->OnTargetFramerateUpdated(target_framerate);
-
ConfigureQualityScaler();
}
@@ -821,7 +806,7 @@
TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
"Encode");
- overuse_detector_->FrameCaptured(out_frame, time_when_posted_us);
+ overuse_detector_->FrameCaptured(out_frame.width(), out_frame.height());
video_sender_.AddVideoFrame(out_frame, nullptr);
}
@@ -847,12 +832,21 @@
EncodedImageCallback::Result result =
sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
- int64_t time_sent_us = rtc::TimeMicros();
- uint32_t timestamp = encoded_image._timeStamp;
+ int64_t capture_time_us =
+ encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
+ int64_t encode_duration_us;
+ if (encoded_image.timing_.flags != TimingFrameFlags::kInvalid) {
+ encode_duration_us = rtc::kNumMicrosecsPerMillisec *
+ (encoded_image.timing_.encode_finish_ms -
+ encoded_image.timing_.encode_start_ms);
+ } else {
+ encode_duration_us = -1;
+ }
const int qp = encoded_image.qp_;
- encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] {
+ encoder_queue_.PostTask([this, capture_time_us, encode_duration_us, qp] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- overuse_detector_->FrameSent(timestamp, time_sent_us);
+ if (encode_duration_us >= 0)
+ overuse_detector_->FrameEncoded(capture_time_us, encode_duration_us);
if (quality_scaler_ && qp >= 0)
quality_scaler_->ReportQP(qp);
});
@@ -1003,8 +997,6 @@
if (requested_framerate == -1)
return;
RTC_DCHECK_NE(max_framerate_, -1);
- overuse_detector_->OnTargetFramerateUpdated(
- std::min(max_framerate_, requested_framerate));
GetAdaptCounter().IncrementFramerate(reason);
break;
}
@@ -1088,11 +1080,8 @@
const int requested_framerate =
source_proxy_->RequestHigherFramerateThan(fps);
if (requested_framerate == -1) {
- overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
return;
}
- overuse_detector_->OnTargetFramerateUpdated(
- std::min(max_framerate_, requested_framerate));
GetAdaptCounter().DecrementFramerate(reason);
break;
}
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 5afbe27..02d6d3e 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -63,39 +63,11 @@
rtc::Event* const event_;
};
-class CpuOveruseDetectorProxy : public OveruseFrameDetector {
- public:
- CpuOveruseDetectorProxy(const CpuOveruseOptions& options,
- AdaptationObserverInterface* overuse_observer,
- EncodedFrameObserver* encoder_timing_,
- CpuOveruseMetricsObserver* metrics_observer)
- : OveruseFrameDetector(options,
- overuse_observer,
- encoder_timing_,
- metrics_observer),
- last_target_framerate_fps_(-1) {}
- virtual ~CpuOveruseDetectorProxy() {}
-
- void OnTargetFramerateUpdated(int framerate_fps) override {
- rtc::CritScope cs(&lock_);
- last_target_framerate_fps_ = framerate_fps;
- OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
- }
-
- int GetLastTargetFramerate() {
- rtc::CritScope cs(&lock_);
- return last_target_framerate_fps_;
- }
-
- private:
- rtc::CriticalSection lock_;
- int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
-};
-
class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
public:
- VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
- const VideoSendStream::Config::EncoderSettings& settings)
+ VideoStreamEncoderUnderTest(
+ SendStatisticsProxy* stats_proxy,
+ const VideoSendStream::Config::EncoderSettings& settings)
: VideoStreamEncoder(
1 /* number_of_cores */,
stats_proxy,
@@ -103,7 +75,7 @@
nullptr /* pre_encode_callback */,
nullptr /* encoder_timing */,
std::unique_ptr<OveruseFrameDetector>(
- overuse_detector_proxy_ = new CpuOveruseDetectorProxy(
+ overuse_detector_ = new OveruseFrameDetector(
GetCpuOveruseOptions(settings.full_overuse_time),
this,
nullptr,
@@ -136,7 +108,7 @@
void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
- CpuOveruseDetectorProxy* overuse_detector_proxy_;
+ OveruseFrameDetector* overuse_detector_;
};
class VideoStreamFactory
@@ -2238,126 +2210,6 @@
video_stream_encoder_->Stop();
}
-TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
- const int kFrameWidth = 1280;
- const int kFrameHeight = 720;
- const int kFramerate = 24;
-
- video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
- test::FrameForwarder source;
- video_stream_encoder_->SetSource(
- &source, VideoSendStream::DegradationPreference::kMaintainResolution);
-
- // Insert a single frame, triggering initial configuration.
- source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
- video_stream_encoder_->WaitUntilTaskQueueIsIdle();
-
- EXPECT_EQ(
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
- kDefaultFramerate);
-
- // Trigger reconfigure encoder (without resetting the entire instance).
- VideoEncoderConfig video_encoder_config;
- video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
- video_encoder_config.number_of_streams = 1;
- video_encoder_config.video_stream_factory =
- new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
- video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
- kMaxPayloadLength, false);
- video_stream_encoder_->WaitUntilTaskQueueIsIdle();
-
- // Detector should be updated with fps limit from codec config.
- EXPECT_EQ(
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
- kFramerate);
-
- // Trigger overuse, max framerate should be reduced.
- VideoSendStream::Stats stats = stats_proxy_->GetStats();
- stats.input_frame_rate = kFramerate;
- stats_proxy_->SetMockStats(stats);
- video_stream_encoder_->TriggerCpuOveruse();
- video_stream_encoder_->WaitUntilTaskQueueIsIdle();
- int adapted_framerate =
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
- EXPECT_LT(adapted_framerate, kFramerate);
-
- // Trigger underuse, max framerate should go back to codec configured fps.
- // Set extra low fps, to make sure it's actually reset, not just incremented.
- stats = stats_proxy_->GetStats();
- stats.input_frame_rate = adapted_framerate / 2;
- stats_proxy_->SetMockStats(stats);
- video_stream_encoder_->TriggerCpuNormalUsage();
- video_stream_encoder_->WaitUntilTaskQueueIsIdle();
- EXPECT_EQ(
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
- kFramerate);
-
- video_stream_encoder_->Stop();
-}
-
-TEST_F(VideoStreamEncoderTest,
- OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
- const int kFrameWidth = 1280;
- const int kFrameHeight = 720;
- const int kLowFramerate = 15;
- const int kHighFramerate = 25;
-
- video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
- test::FrameForwarder source;
- video_stream_encoder_->SetSource(
- &source, VideoSendStream::DegradationPreference::kMaintainResolution);
-
- // Trigger initial configuration.
- VideoEncoderConfig video_encoder_config;
- video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
- video_encoder_config.number_of_streams = 1;
- video_encoder_config.video_stream_factory =
- new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
- source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
- video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
- kMaxPayloadLength, false);
- video_stream_encoder_->WaitUntilTaskQueueIsIdle();
-
- EXPECT_EQ(
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
- kLowFramerate);
-
- // Trigger overuse, max framerate should be reduced.
- VideoSendStream::Stats stats = stats_proxy_->GetStats();
- stats.input_frame_rate = kLowFramerate;
- stats_proxy_->SetMockStats(stats);
- video_stream_encoder_->TriggerCpuOveruse();
- video_stream_encoder_->WaitUntilTaskQueueIsIdle();
- int adapted_framerate =
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
- EXPECT_LT(adapted_framerate, kLowFramerate);
-
- // Reconfigure the encoder with a new (higher max framerate), max fps should
- // still respect the adaptation.
- video_encoder_config.video_stream_factory =
- new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
- source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
- video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
- kMaxPayloadLength, false);
- video_stream_encoder_->WaitUntilTaskQueueIsIdle();
-
- EXPECT_EQ(
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
- adapted_framerate);
-
- // Trigger underuse, max framerate should go back to codec configured fps.
- stats = stats_proxy_->GetStats();
- stats.input_frame_rate = adapted_framerate;
- stats_proxy_->SetMockStats(stats);
- video_stream_encoder_->TriggerCpuNormalUsage();
- video_stream_encoder_->WaitUntilTaskQueueIsIdle();
- EXPECT_EQ(
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
- kHighFramerate);
-
- video_stream_encoder_->Stop();
-}
-
TEST_F(VideoStreamEncoderTest,
OveruseDetectorUpdatedOnDegradationPreferenceChange) {
const int kFrameWidth = 1280;
@@ -2380,9 +2232,7 @@
kMaxPayloadLength, false);
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
- EXPECT_EQ(
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
- kFramerate);
+ EXPECT_GT(source.sink_wants().max_framerate_fps, kFramerate);
// Trigger overuse, max framerate should be reduced.
VideoSendStream::Stats stats = stats_proxy_->GetStats();
@@ -2390,8 +2240,8 @@
stats_proxy_->SetMockStats(stats);
video_stream_encoder_->TriggerCpuOveruse();
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
- int adapted_framerate =
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
+
+ int adapted_framerate = source.sink_wants().max_framerate_fps;
EXPECT_LT(adapted_framerate, kFramerate);
// Change degradation preference to not enable framerate scaling. Target
@@ -2399,9 +2249,7 @@
video_stream_encoder_->SetSource(
&source, VideoSendStream::DegradationPreference::kMaintainFramerate);
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
- EXPECT_EQ(
- video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
- kFramerate);
+ EXPECT_GT(source.sink_wants().max_framerate_fps, kFramerate);
video_stream_encoder_->Stop();
}
