Restrict usage of resolution bitrate limits to singlecast
Bug: none
Change-Id: I4d0726d45a517b51eae124dc23e533910ede7cc7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/203262
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33061}
diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn
index c5afb02..b908ac3 100644
--- a/video/adaptation/BUILD.gn
+++ b/video/adaptation/BUILD.gn
@@ -75,6 +75,7 @@
defines = []
sources = [
+ "bitrate_constraint_unittest.cc",
"overuse_frame_detector_unittest.cc",
"pixel_limit_resource_unittest.cc",
"quality_scaler_resource_unittest.cc",
diff --git a/video/adaptation/bitrate_constraint.cc b/video/adaptation/bitrate_constraint.cc
index 1061c45..28b5058 100644
--- a/video/adaptation/bitrate_constraint.cc
+++ b/video/adaptation/bitrate_constraint.cc
@@ -10,13 +10,32 @@
#include <string>
#include <utility>
+#include <vector>
#include "call/adaptation/video_stream_adapter.h"
#include "rtc_base/synchronization/sequence_checker.h"
#include "video/adaptation/bitrate_constraint.h"
+#include "video/adaptation/video_stream_encoder_resource_manager.h"
namespace webrtc {
+namespace {
+bool IsSimulcast(const VideoEncoderConfig& encoder_config) {
+ const std::vector<VideoStream>& simulcast_layers =
+ encoder_config.simulcast_layers;
+
+ bool is_simulcast = simulcast_layers.size() > 1;
+ bool is_lowest_layer_active = simulcast_layers[0].active;
+ int num_active_layers =
+ std::count_if(simulcast_layers.begin(), simulcast_layers.end(),
+ [](const VideoStream& layer) { return layer.active; });
+
+ // We can't distinguish between simulcast and singlecast when only the
+ // lowest spatial layer is active. Treat this case as simulcast.
+ return is_simulcast && (num_active_layers > 1 || is_lowest_layer_active);
+}
+} // namespace
+
BitrateConstraint::BitrateConstraint()
: encoder_settings_(absl::nullopt),
encoder_target_bitrate_bps_(absl::nullopt) {
@@ -42,19 +61,35 @@
RTC_DCHECK_RUN_ON(&sequence_checker_);
// Make sure bitrate limits are not violated.
if (DidIncreaseResolution(restrictions_before, restrictions_after)) {
+ if (!encoder_settings_.has_value()) {
+ return true;
+ }
+
uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0);
+ if (bitrate_bps == 0) {
+ return true;
+ }
+
+ if (IsSimulcast(encoder_settings_->encoder_config())) {
+ // Resolution bitrate limits usage is restricted to singlecast.
+ return true;
+ }
+
+ absl::optional<uint32_t> current_frame_size_px =
+ VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels(
+ encoder_settings_->video_codec());
+ if (!current_frame_size_px.has_value()) {
+ return true;
+ }
+
absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
- encoder_settings_.has_value()
- ? encoder_settings_->encoder_info()
- .GetEncoderBitrateLimitsForResolution(
- // Need some sort of expected resulting pixels to be used
- // instead of unrestricted.
- GetHigherResolutionThan(
- input_state.frame_size_pixels().value()))
- : absl::nullopt;
- if (bitrate_limits.has_value() && bitrate_bps != 0) {
- RTC_DCHECK_GE(bitrate_limits->frame_size_pixels,
- input_state.frame_size_pixels().value());
+ encoder_settings_->encoder_info().GetEncoderBitrateLimitsForResolution(
+ // Need some sort of expected resulting pixels to be used
+ // instead of unrestricted.
+ GetHigherResolutionThan(*current_frame_size_px));
+
+ if (bitrate_limits.has_value()) {
+ RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, *current_frame_size_px);
return bitrate_bps >=
static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
}
diff --git a/video/adaptation/bitrate_constraint_unittest.cc b/video/adaptation/bitrate_constraint_unittest.cc
new file mode 100644
index 0000000..e60418f
--- /dev/null
+++ b/video/adaptation/bitrate_constraint_unittest.cc
@@ -0,0 +1,223 @@
+/*
+ * Copyright 2021 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/adaptation/bitrate_constraint.h"
+
+#include <limits>
+#include <utility>
+#include <vector>
+
+#include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/encoder_settings.h"
+#include "call/adaptation/video_source_restrictions.h"
+#include "call/adaptation/video_stream_input_state.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+using ResolutionBitrateLimits = VideoEncoder::ResolutionBitrateLimits;
+
+namespace {
+
+void FillCodecConfig(VideoCodec* video_codec,
+ VideoEncoderConfig* encoder_config,
+ int width_px,
+ int height_px,
+ std::vector<bool> active_flags) {
+ size_t num_layers = active_flags.size();
+ video_codec->codecType = kVideoCodecVP8;
+ video_codec->numberOfSimulcastStreams = num_layers;
+
+ encoder_config->number_of_streams = num_layers;
+ encoder_config->simulcast_layers.resize(num_layers);
+
+ for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
+ int layer_width_px = width_px >> (num_layers - 1 - layer_idx);
+ int layer_height_px = height_px >> (num_layers - 1 - layer_idx);
+
+ video_codec->simulcastStream[layer_idx].active = active_flags[layer_idx];
+ video_codec->simulcastStream[layer_idx].width = layer_width_px;
+ video_codec->simulcastStream[layer_idx].height = layer_height_px;
+
+ encoder_config->simulcast_layers[layer_idx].active =
+ active_flags[layer_idx];
+ encoder_config->simulcast_layers[layer_idx].width = layer_width_px;
+ encoder_config->simulcast_layers[layer_idx].height = layer_height_px;
+ }
+}
+
+VideoEncoder::EncoderInfo MakeEncoderInfo() {
+ VideoEncoder::EncoderInfo encoder_info;
+ encoder_info.resolution_bitrate_limits = std::vector<ResolutionBitrateLimits>(
+ {ResolutionBitrateLimits(640 * 360, 500000, 0, 5000000),
+ ResolutionBitrateLimits(1280 * 720, 1000000, 0, 5000000),
+ ResolutionBitrateLimits(1920 * 1080, 2000000, 0, 5000000)});
+ return encoder_info;
+}
+} // namespace
+
+TEST(BitrateConstraintTest, AdaptUpAllowedAtSinglecastIfBitrateIsEnough) {
+ VideoCodec video_codec;
+ VideoEncoderConfig encoder_config;
+ FillCodecConfig(&video_codec, &encoder_config,
+ /*width_px=*/640, /*height_px=*/360,
+ /*active_flags=*/{true});
+
+ EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+ video_codec);
+
+ BitrateConstraint bitrate_constraint;
+ bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+ bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000);
+
+ VideoSourceRestrictions restrictions_before(
+ /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+ /*max_frame_rate=*/30);
+ VideoSourceRestrictions restrictions_after(
+ /*max_pixels_per_frame=*/1280 * 720,
+ /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+ EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed(
+ VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest, AdaptUpDisallowedAtSinglecastIfBitrateIsNotEnough) {
+ VideoCodec video_codec;
+ VideoEncoderConfig encoder_config;
+ FillCodecConfig(&video_codec, &encoder_config,
+ /*width_px=*/640, /*height_px=*/360,
+ /*active_flags=*/{true});
+
+ EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+ video_codec);
+
+ BitrateConstraint bitrate_constraint;
+ bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+ // 1 bps less than needed for 720p.
+ bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1);
+
+ VideoSourceRestrictions restrictions_before(
+ /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+ /*max_frame_rate=*/30);
+ VideoSourceRestrictions restrictions_after(
+ /*max_pixels_per_frame=*/1280 * 720,
+ /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+ EXPECT_FALSE(bitrate_constraint.IsAdaptationUpAllowed(
+ VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest,
+ AdaptUpAllowedAtSinglecastUpperLayerActiveIfBitrateIsEnough) {
+ VideoCodec video_codec;
+ VideoEncoderConfig encoder_config;
+ FillCodecConfig(&video_codec, &encoder_config,
+ /*width_px=*/640, /*height_px=*/360,
+ /*active_flags=*/{false, true});
+
+ EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+ video_codec);
+
+ BitrateConstraint bitrate_constraint;
+ bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+ bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000);
+
+ VideoSourceRestrictions restrictions_before(
+ /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+ /*max_frame_rate=*/30);
+ VideoSourceRestrictions restrictions_after(
+ /*max_pixels_per_frame=*/1280 * 720,
+ /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+ EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed(
+ VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest,
+ AdaptUpDisallowedAtSinglecastUpperLayerActiveIfBitrateIsNotEnough) {
+ VideoCodec video_codec;
+ VideoEncoderConfig encoder_config;
+ FillCodecConfig(&video_codec, &encoder_config,
+ /*width_px=*/640, /*height_px=*/360,
+ /*active_flags=*/{false, true});
+
+ EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+ video_codec);
+
+ BitrateConstraint bitrate_constraint;
+ bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+ // 1 bps less than needed for 720p.
+ bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1);
+
+ VideoSourceRestrictions restrictions_before(
+ /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+ /*max_frame_rate=*/30);
+ VideoSourceRestrictions restrictions_after(
+ /*max_pixels_per_frame=*/1280 * 720,
+ /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+ EXPECT_FALSE(bitrate_constraint.IsAdaptationUpAllowed(
+ VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest,
+ AdaptUpAllowedAtSinglecastLowestLayerActiveIfBitrateIsNotEnough) {
+ VideoCodec video_codec;
+ VideoEncoderConfig encoder_config;
+ FillCodecConfig(&video_codec, &encoder_config,
+ /*width_px=*/640, /*height_px=*/360,
+ /*active_flags=*/{true, false});
+
+ EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+ video_codec);
+
+ BitrateConstraint bitrate_constraint;
+ bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+ // 1 bps less than needed for 720p.
+ bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1);
+
+ VideoSourceRestrictions restrictions_before(
+ /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+ /*max_frame_rate=*/30);
+ VideoSourceRestrictions restrictions_after(
+ /*max_pixels_per_frame=*/1280 * 720,
+ /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+ EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed(
+ VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest, AdaptUpAllowedAtSimulcastIfBitrateIsNotEnough) {
+ VideoCodec video_codec;
+ VideoEncoderConfig encoder_config;
+ FillCodecConfig(&video_codec, &encoder_config,
+ /*width_px=*/640, /*height_px=*/360,
+ /*active_flags=*/{true, true});
+
+ EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+ video_codec);
+
+ BitrateConstraint bitrate_constraint;
+ bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+ // 1 bps less than needed for 720p.
+ bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1);
+
+ VideoSourceRestrictions restrictions_before(
+ /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+ /*max_frame_rate=*/30);
+ VideoSourceRestrictions restrictions_after(
+ /*max_pixels_per_frame=*/1280 * 720,
+ /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+ EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed(
+ VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+} // namespace webrtc
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc
index 81d23a1..96f888d 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.cc
+++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -64,30 +64,6 @@
RTC_CHECK_NOTREACHED();
}
-absl::optional<uint32_t> GetSingleActiveStreamPixels(const VideoCodec& codec) {
- int num_active = 0;
- absl::optional<uint32_t> pixels;
- if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
- for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
- if (codec.spatialLayers[i].active) {
- ++num_active;
- pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
- }
- }
- } else {
- for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
- if (codec.simulcastStream[i].active) {
- ++num_active;
- pixels =
- codec.simulcastStream[i].width * codec.simulcastStream[i].height;
- }
- }
- }
- if (num_active > 1)
- return absl::nullopt;
- return pixels;
-}
-
std::vector<bool> GetActiveLayersFlags(const VideoCodec& codec) {
std::vector<bool> flags;
if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
@@ -188,7 +164,7 @@
last_active_flags_ = active_flags;
last_input_width_ = codec.width;
last_input_height_ = codec.height;
- single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec);
+ single_active_stream_pixels_ = GetSingleActiveLayerPixels(codec);
}
void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
@@ -703,4 +679,31 @@
stream_adapter_->ClearRestrictions();
quality_rampup_experiment_.reset();
}
+
+absl::optional<uint32_t>
+VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels(
+ const VideoCodec& codec) {
+ int num_active = 0;
+ absl::optional<uint32_t> pixels;
+ if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
+ for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
+ if (codec.spatialLayers[i].active) {
+ ++num_active;
+ pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
+ }
+ }
+ } else {
+ for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
+ if (codec.simulcastStream[i].active) {
+ ++num_active;
+ pixels =
+ codec.simulcastStream[i].width * codec.simulcastStream[i].height;
+ }
+ }
+ }
+ if (num_active > 1)
+ return absl::nullopt;
+ return pixels;
+}
+
} // namespace webrtc
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h
index 7e458a9..d6b9dd1 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.h
+++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -146,6 +146,9 @@
// QualityRampUpExperimentListener implementation.
void OnQualityRampUp() override;
+ static absl::optional<uint32_t> GetSingleActiveLayerPixels(
+ const VideoCodec& codec);
+
private:
class InitialFrameDropper;
