Update adaptation stats to support degradations in both resolution and framerate.
Add AdaptCounter class which holds the number of downgrade counts per degradation way (resolution/fps) and reason (cpu/quality).
BUG=webrtc:7607
Review-Url: https://codereview.webrtc.org/2871623002
Cr-Commit-Position: refs/heads/master@{#18156}
diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc
index d2f46e6..eb39bf7 100644
--- a/webrtc/video/vie_encoder_unittest.cc
+++ b/webrtc/video/vie_encoder_unittest.cc
@@ -139,6 +139,11 @@
return adaptation_enabled_;
}
+ rtc::VideoSinkWants last_wants() const {
+ rtc::CritScope cs(&crit_);
+ return last_wants_;
+ }
+
void IncomingCapturedFrame(const VideoFrame& video_frame) override {
int cropped_width = 0;
int cropped_height = 0;
@@ -163,14 +168,15 @@
void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
const rtc::VideoSinkWants& wants) override {
rtc::CritScope cs(&crit_);
+ last_wants_ = sink_wants();
adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
wants.max_pixel_count,
wants.max_framerate_fps);
test::FrameForwarder::AddOrUpdateSink(sink, wants);
}
-
cricket::VideoAdapter adapter_;
bool adaptation_enabled_ GUARDED_BY(crit_);
+ rtc::VideoSinkWants last_wants_ GUARDED_BY(crit_);
};
class MockableSendStatisticsProxy : public SendStatisticsProxy {
@@ -281,16 +287,41 @@
}
void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
- EXPECT_FALSE(wants.target_pixel_count);
- EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
+ EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
+ EXPECT_FALSE(wants.target_pixel_count);
}
- void VerifyResolutionLimitationLessThan(const rtc::VideoSinkWants& wants,
- int pixel_count) {
+ void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
+ const rtc::VideoSinkWants& wants2) {
+ EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
+ EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
+ }
+
+ void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
+ const rtc::VideoSinkWants& wants2) {
+ EXPECT_EQ(std::numeric_limits<int>::max(), wants1.max_framerate_fps);
+ EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
+ EXPECT_GT(wants1.max_pixel_count, 0);
+ }
+
+ void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
+ const rtc::VideoSinkWants& wants2) {
+ EXPECT_EQ(std::numeric_limits<int>::max(), wants1.max_framerate_fps);
+ EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
+ }
+
+ void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
+ int pixel_count) {
+ EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
EXPECT_LT(wants.max_pixel_count, pixel_count);
EXPECT_GT(wants.max_pixel_count, 0);
- EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
+ }
+
+ void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
+ EXPECT_LT(wants.max_framerate_fps, fps);
+ EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
+ EXPECT_FALSE(wants.target_pixel_count);
}
class TestEncoder : public test::FakeEncoder {
@@ -946,6 +977,7 @@
video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sink_.WaitForEncodedFrame(1);
VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
EXPECT_FALSE(stats.cpu_limited_resolution);
EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
@@ -954,6 +986,7 @@
video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sink_.WaitForEncodedFrame(2);
stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
EXPECT_TRUE(stats.cpu_limited_resolution);
EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
@@ -966,6 +999,7 @@
new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sink_.WaitForEncodedFrame(3);
stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
EXPECT_TRUE(stats.cpu_limited_resolution);
EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
@@ -977,6 +1011,7 @@
new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sink_.WaitForEncodedFrame(4);
stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
EXPECT_FALSE(stats.cpu_limited_resolution);
EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
@@ -988,6 +1023,7 @@
new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sink_.WaitForEncodedFrame(5);
stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
EXPECT_TRUE(stats.cpu_limited_resolution);
EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
@@ -996,6 +1032,7 @@
new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sink_.WaitForEncodedFrame(6);
stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
EXPECT_FALSE(stats.cpu_limited_resolution);
EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
@@ -1138,6 +1175,7 @@
sink_.WaitForEncodedFrame(sequence++);
stats = stats_proxy_->GetStats();
EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
// Set cpu adaptation by frame dropping.
@@ -1150,10 +1188,11 @@
stats = stats_proxy_->GetStats();
// Not adapted at first.
EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
// Force an input frame rate to be available, or the adaptation call won't
- // know what framerate to adapt form.
+ // know what framerate to adapt from.
VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
mock_stats.input_frame_rate = 30;
stats_proxy_->SetMockStats(mock_stats);
@@ -1166,7 +1205,8 @@
// Framerate now adapted.
stats = stats_proxy_->GetStats();
- EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_TRUE(stats.cpu_limited_framerate);
EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
// Disable CPU adaptation.
@@ -1314,7 +1354,7 @@
// Trigger adapt down, expect scaled down resolution.
vie_encoder_->TriggerCpuOveruse();
- VerifyResolutionLimitationLessThan(source.sink_wants(), kWidth * kHeight);
+ VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1366,18 +1406,45 @@
source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sink_.WaitForEncodedFrame(kWidth, kHeight);
VerifyNoLimitation(source.sink_wants());
- EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
// Trigger adapt up, expect no change.
vie_encoder_->TriggerCpuNormalUsage();
VerifyNoLimitation(source.sink_wants());
- EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
vie_encoder_->Stop();
}
+TEST_F(ViEEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Enable kDegradationDisabled preference, no initial limitation.
+ test::FrameForwarder source;
+ vie_encoder_->SetSource(
+ &source, VideoSendStream::DegradationPreference::kDegradationDisabled);
+
+ source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ VerifyNoLimitation(source.sink_wants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no change.
+ vie_encoder_->TriggerQualityHigh();
+ VerifyNoLimitation(source.sink_wants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ vie_encoder_->Stop();
+}
+
TEST_F(ViEEncoderTest, AdaptsResolutionForLowQuality_MaintainFramerateMode) {
const int kWidth = 1280;
const int kHeight = 720;
@@ -1399,7 +1466,7 @@
vie_encoder_->TriggerQualityLow();
source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sink_.WaitForEncodedFrame(2);
- VerifyResolutionLimitationLessThan(source.sink_wants(), kWidth * kHeight);
+ VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
@@ -1413,6 +1480,47 @@
vie_encoder_->Stop();
}
+TEST_F(ViEEncoderTest, AdaptsFramerateForLowQuality_MaintainResolutionMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ const int kInputFps = 30;
+ vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kInputFps;
+ stats_proxy_->SetMockStats(stats);
+
+ // Expect no scaling to begin with (preference: kMaintainFramerate).
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(1);
+ VerifyNoLimitation(video_source_.sink_wants());
+
+ // Trigger adapt down, expect scaled down resolution.
+ vie_encoder_->TriggerQualityLow();
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(2);
+ VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
+
+ // Enable kMaintainResolution preference.
+ test::FrameForwarder new_video_source;
+ vie_encoder_->SetSource(
+ &new_video_source,
+ VideoSendStream::DegradationPreference::kMaintainResolution);
+ VerifyNoLimitation(new_video_source.sink_wants());
+
+ // Trigger adapt down, expect reduced framerate.
+ vie_encoder_->TriggerQualityLow();
+ new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(3);
+ VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
+
+ // Trigger adapt up, expect no restriction.
+ vie_encoder_->TriggerQualityHigh();
+ VerifyNoLimitation(new_video_source.sink_wants());
+
+ vie_encoder_->Stop();
+}
+
TEST_F(ViEEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
const int kWidth = 1280;
const int kHeight = 720;
@@ -1461,7 +1569,7 @@
&source, VideoSendStream::DegradationPreference::kMaintainFramerate);
source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
- sink_.WaitForEncodedFrame(1);
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
VerifyNoLimitation(source.sink_wants());
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1470,14 +1578,14 @@
vie_encoder_->TriggerCpuOveruse();
source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sink_.WaitForEncodedFrame(2);
- VerifyResolutionLimitationLessThan(source.sink_wants(), kWidth * kHeight);
+ VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
// Trigger adapt up, expect no restriction.
vie_encoder_->TriggerCpuNormalUsage();
source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
- sink_.WaitForEncodedFrame(3);
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
VerifyNoLimitation(source.sink_wants());
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1486,12 +1594,14 @@
vie_encoder_->TriggerCpuOveruse();
source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sink_.WaitForEncodedFrame(4);
- VerifyResolutionLimitationLessThan(source.sink_wants(), kWidth * kHeight);
+ VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
// Trigger adapt up, expect no restriction.
vie_encoder_->TriggerCpuNormalUsage();
+ source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
VerifyNoLimitation(source.sink_wants());
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1523,8 +1633,7 @@
vie_encoder_->TriggerCpuOveruse();
source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sink_.WaitForEncodedFrame(2);
- VerifyResolutionLimitationLessThan(source.sink_wants(), kWidth * kHeight);
- rtc::VideoSinkWants last_wants = source.sink_wants();
+ VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1534,8 +1643,8 @@
vie_encoder_->TriggerCpuOveruse();
source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sink_.WaitForEncodedFrame(3);
- EXPECT_LT(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
- last_wants = source.sink_wants();
+ VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
+ rtc::VideoSinkWants last_wants = source.sink_wants();
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1545,7 +1654,7 @@
vie_encoder_->TriggerCpuOveruse();
source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sink_.WaitForEncodedFrame(4);
- EXPECT_EQ(last_wants.max_pixel_count, source.sink_wants().max_pixel_count);
+ VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1555,8 +1664,7 @@
vie_encoder_->TriggerQualityLow();
source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sink_.WaitForEncodedFrame(5);
- EXPECT_LT(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
- last_wants = source.sink_wants();
+ VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1566,8 +1674,7 @@
vie_encoder_->TriggerCpuNormalUsage();
source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sink_.WaitForEncodedFrame(6);
- EXPECT_GT(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
- last_wants = source.sink_wants();
+ VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1577,7 +1684,7 @@
vie_encoder_->TriggerCpuNormalUsage();
source.IncomingCapturedFrame(CreateFrame(7, kWidth, kHeight));
sink_.WaitForEncodedFrame(7);
- EXPECT_GT(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
+ VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
last_wants = source.sink_wants();
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
@@ -1588,7 +1695,7 @@
vie_encoder_->TriggerCpuNormalUsage();
source.IncomingCapturedFrame(CreateFrame(8, kWidth, kHeight));
sink_.WaitForEncodedFrame(8);
- EXPECT_EQ(last_wants.max_pixel_count, source.sink_wants().max_pixel_count);
+ VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
@@ -1598,7 +1705,7 @@
vie_encoder_->TriggerQualityHigh();
source.IncomingCapturedFrame(CreateFrame(9, kWidth, kHeight));
sink_.WaitForEncodedFrame(kWidth, kHeight);
- EXPECT_GT(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
+ VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
VerifyNoLimitation(source.sink_wants());
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
@@ -1730,7 +1837,7 @@
vie_encoder_->Stop();
}
-TEST_F(ViEEncoderTest, NrOfDroppedFramesLimitedWhenBitrateIsTooLow) {
+TEST_F(ViEEncoderTest, NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
const int kTooLowBitrateForFrameSizeBps = 10000;
vie_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
const int kWidth = 640;
@@ -1774,6 +1881,7 @@
const int kHeight = 360;
fake_encoder_.SetQualityScaling(false);
vie_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
+
// Force quality scaler reconfiguration by resetting the source.
vie_encoder_->SetSource(&video_source_,
VideoSendStream::DegradationPreference::kBalanced);
