| /* |
| * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "modules/video_coding/svc/svc_rate_allocator.h" |
| |
| #include <algorithm> |
| #include <cmath> |
| #include <cstddef> |
| #include <numeric> |
| #include <vector> |
| |
| #include "absl/container/inlined_vector.h" |
| #include "modules/video_coding/svc/create_scalability_structure.h" |
| #include "rtc_base/checks.h" |
| |
| namespace webrtc { |
| namespace { |
| |
| constexpr float kSpatialLayeringRateScalingFactor = 0.55f; |
| constexpr float kTemporalLayeringRateScalingFactor = 0.55f; |
| |
| struct ActiveSpatialLayers { |
| size_t first = 0; |
| size_t num = 0; |
| }; |
| |
| ActiveSpatialLayers GetActiveSpatialLayers(const VideoCodec& codec, |
| size_t num_spatial_layers) { |
| ActiveSpatialLayers active; |
| for (active.first = 0; active.first < num_spatial_layers; ++active.first) { |
| if (codec.spatialLayers[active.first].active) { |
| break; |
| } |
| } |
| |
| size_t last_active_layer = active.first; |
| for (; last_active_layer < num_spatial_layers; ++last_active_layer) { |
| if (!codec.spatialLayers[last_active_layer].active) { |
| break; |
| } |
| } |
| active.num = last_active_layer - active.first; |
| |
| return active; |
| } |
| |
| std::vector<DataRate> AdjustAndVerify( |
| const VideoCodec& codec, |
| size_t first_active_layer, |
| const std::vector<DataRate>& spatial_layer_rates) { |
| std::vector<DataRate> adjusted_spatial_layer_rates; |
| // Keep track of rate that couldn't be applied to the previous layer due to |
| // max bitrate constraint, try to pass it forward to the next one. |
| DataRate excess_rate = DataRate::Zero(); |
| for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) { |
| DataRate min_rate = DataRate::KilobitsPerSec( |
| codec.spatialLayers[first_active_layer + sl_idx].minBitrate); |
| DataRate max_rate = DataRate::KilobitsPerSec( |
| codec.spatialLayers[first_active_layer + sl_idx].maxBitrate); |
| |
| DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate; |
| if (layer_rate < min_rate) { |
| // Not enough rate to reach min bitrate for desired number of layers, |
| // abort allocation. |
| if (spatial_layer_rates.size() == 1) { |
| return spatial_layer_rates; |
| } |
| return adjusted_spatial_layer_rates; |
| } |
| |
| if (layer_rate <= max_rate) { |
| excess_rate = DataRate::Zero(); |
| adjusted_spatial_layer_rates.push_back(layer_rate); |
| } else { |
| excess_rate = layer_rate - max_rate; |
| adjusted_spatial_layer_rates.push_back(max_rate); |
| } |
| } |
| |
| return adjusted_spatial_layer_rates; |
| } |
| |
| static std::vector<DataRate> SplitBitrate(size_t num_layers, |
| DataRate total_bitrate, |
| float rate_scaling_factor) { |
| std::vector<DataRate> bitrates; |
| |
| double denominator = 0.0; |
| for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) { |
| denominator += std::pow(rate_scaling_factor, layer_idx); |
| } |
| |
| double numerator = std::pow(rate_scaling_factor, num_layers - 1); |
| for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) { |
| bitrates.push_back(numerator * total_bitrate / denominator); |
| numerator /= rate_scaling_factor; |
| } |
| |
| const DataRate sum = |
| std::accumulate(bitrates.begin(), bitrates.end(), DataRate::Zero()); |
| |
| // Keep the sum of split bitrates equal to the total bitrate by adding or |
| // subtracting bits, which were lost due to rounding, to the latest layer. |
| if (total_bitrate > sum) { |
| bitrates.back() += total_bitrate - sum; |
| } else if (total_bitrate < sum) { |
| bitrates.back() -= sum - total_bitrate; |
| } |
| |
| return bitrates; |
| } |
| |
| // Returns the minimum bitrate needed for `num_active_layers` spatial layers to |
| // become active using the configuration specified by `codec`. |
| DataRate FindLayerTogglingThreshold(const VideoCodec& codec, |
| size_t first_active_layer, |
| size_t num_active_layers) { |
| if (num_active_layers == 1) { |
| return DataRate::KilobitsPerSec(codec.spatialLayers[0].minBitrate); |
| } |
| |
| if (codec.mode == VideoCodecMode::kRealtimeVideo) { |
| DataRate lower_bound = DataRate::Zero(); |
| DataRate upper_bound = DataRate::Zero(); |
| if (num_active_layers > 1) { |
| for (size_t i = 0; i < num_active_layers - 1; ++i) { |
| lower_bound += DataRate::KilobitsPerSec( |
| codec.spatialLayers[first_active_layer + i].minBitrate); |
| upper_bound += DataRate::KilobitsPerSec( |
| codec.spatialLayers[first_active_layer + i].maxBitrate); |
| } |
| } |
| upper_bound += DataRate::KilobitsPerSec( |
| codec.spatialLayers[first_active_layer + num_active_layers - 1] |
| .minBitrate); |
| |
| // Do a binary search until upper and lower bound is the highest bitrate for |
| // `num_active_layers` - 1 layers and lowest bitrate for `num_active_layers` |
| // layers respectively. |
| while (upper_bound - lower_bound > DataRate::BitsPerSec(1)) { |
| DataRate try_rate = (lower_bound + upper_bound) / 2; |
| if (AdjustAndVerify(codec, first_active_layer, |
| SplitBitrate(num_active_layers, try_rate, |
| kSpatialLayeringRateScalingFactor)) |
| .size() == num_active_layers) { |
| upper_bound = try_rate; |
| } else { |
| lower_bound = try_rate; |
| } |
| } |
| return upper_bound; |
| } else { |
| DataRate toggling_rate = DataRate::Zero(); |
| for (size_t i = 0; i < num_active_layers - 1; ++i) { |
| toggling_rate += DataRate::KilobitsPerSec( |
| codec.spatialLayers[first_active_layer + i].targetBitrate); |
| } |
| toggling_rate += DataRate::KilobitsPerSec( |
| codec.spatialLayers[first_active_layer + num_active_layers - 1] |
| .minBitrate); |
| return toggling_rate; |
| } |
| } |
| |
| } // namespace |
| |
| SvcRateAllocator::NumLayers SvcRateAllocator::GetNumLayers( |
| const VideoCodec& codec) { |
| NumLayers layers; |
| if (absl::optional<ScalabilityMode> scalability_mode = |
| codec.GetScalabilityMode(); |
| scalability_mode.has_value()) { |
| if (auto structure = CreateScalabilityStructure(*scalability_mode)) { |
| ScalableVideoController::StreamLayersConfig config = |
| structure->StreamConfig(); |
| layers.spatial = config.num_spatial_layers; |
| layers.temporal = config.num_temporal_layers; |
| return layers; |
| } |
| } |
| if (codec.codecType == kVideoCodecVP9) { |
| layers.spatial = codec.VP9().numberOfSpatialLayers; |
| layers.temporal = codec.VP9().numberOfTemporalLayers; |
| return layers; |
| } |
| layers.spatial = 1; |
| layers.temporal = 1; |
| return layers; |
| } |
| |
| SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec) |
| : codec_(codec), |
| num_layers_(GetNumLayers(codec)), |
| experiment_settings_(StableTargetRateExperiment::ParseFromFieldTrials()), |
| cumulative_layer_start_bitrates_(GetLayerStartBitrates(codec)), |
| last_active_layer_count_(0) { |
| RTC_DCHECK_GT(num_layers_.spatial, 0); |
| RTC_DCHECK_LE(num_layers_.spatial, kMaxSpatialLayers); |
| RTC_DCHECK_GT(num_layers_.temporal, 0); |
| RTC_DCHECK_LE(num_layers_.temporal, 3); |
| for (size_t layer_idx = 0; layer_idx < num_layers_.spatial; ++layer_idx) { |
| // Verify min <= target <= max. |
| if (codec.spatialLayers[layer_idx].active) { |
| RTC_DCHECK_GT(codec.spatialLayers[layer_idx].maxBitrate, 0); |
| RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate, |
| codec.spatialLayers[layer_idx].minBitrate); |
| RTC_DCHECK_GE(codec.spatialLayers[layer_idx].targetBitrate, |
| codec.spatialLayers[layer_idx].minBitrate); |
| RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate, |
| codec.spatialLayers[layer_idx].targetBitrate); |
| } |
| } |
| } |
| |
| VideoBitrateAllocation SvcRateAllocator::Allocate( |
| VideoBitrateAllocationParameters parameters) { |
| DataRate total_bitrate = parameters.total_bitrate; |
| if (codec_.maxBitrate != 0) { |
| total_bitrate = |
| std::min(total_bitrate, DataRate::KilobitsPerSec(codec_.maxBitrate)); |
| } |
| |
| if (codec_.spatialLayers[0].targetBitrate == 0) { |
| // Delegate rate distribution to encoder wrapper if bitrate thresholds |
| // are not set. |
| VideoBitrateAllocation bitrate_allocation; |
| bitrate_allocation.SetBitrate(0, 0, total_bitrate.bps()); |
| return bitrate_allocation; |
| } |
| |
| const ActiveSpatialLayers active_layers = |
| GetActiveSpatialLayers(codec_, num_layers_.spatial); |
| size_t num_spatial_layers = active_layers.num; |
| |
| if (num_spatial_layers == 0) { |
| return VideoBitrateAllocation(); // All layers are deactivated. |
| } |
| |
| // Figure out how many spatial layers should be active. |
| if (experiment_settings_.IsEnabled() && |
| parameters.stable_bitrate > DataRate::Zero()) { |
| double hysteresis_factor; |
| if (codec_.mode == VideoCodecMode::kScreensharing) { |
| hysteresis_factor = experiment_settings_.GetScreenshareHysteresisFactor(); |
| } else { |
| hysteresis_factor = experiment_settings_.GetVideoHysteresisFactor(); |
| } |
| |
| DataRate stable_rate = std::min(total_bitrate, parameters.stable_bitrate); |
| // First check if bitrate has grown large enough to enable new layers. |
| size_t num_enabled_with_hysteresis = |
| FindNumEnabledLayers(stable_rate / hysteresis_factor); |
| if (num_enabled_with_hysteresis >= last_active_layer_count_) { |
| num_spatial_layers = num_enabled_with_hysteresis; |
| } else { |
| // We could not enable new layers, check if any should be disabled. |
| num_spatial_layers = |
| std::min(last_active_layer_count_, FindNumEnabledLayers(stable_rate)); |
| } |
| } else { |
| num_spatial_layers = FindNumEnabledLayers(total_bitrate); |
| } |
| last_active_layer_count_ = num_spatial_layers; |
| |
| VideoBitrateAllocation allocation; |
| if (codec_.mode == VideoCodecMode::kRealtimeVideo) { |
| allocation = GetAllocationNormalVideo(total_bitrate, active_layers.first, |
| num_spatial_layers); |
| } else { |
| allocation = GetAllocationScreenSharing(total_bitrate, active_layers.first, |
| num_spatial_layers); |
| } |
| allocation.set_bw_limited(num_spatial_layers < active_layers.num); |
| return allocation; |
| } |
| |
| VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( |
| DataRate total_bitrate, |
| size_t first_active_layer, |
| size_t num_spatial_layers) const { |
| std::vector<DataRate> spatial_layer_rates; |
| if (num_spatial_layers == 0) { |
| // Not enough rate for even the base layer. Force allocation at the total |
| // bitrate anyway. |
| num_spatial_layers = 1; |
| spatial_layer_rates.push_back(total_bitrate); |
| } else { |
| spatial_layer_rates = |
| AdjustAndVerify(codec_, first_active_layer, |
| SplitBitrate(num_spatial_layers, total_bitrate, |
| kSpatialLayeringRateScalingFactor)); |
| RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers); |
| } |
| |
| VideoBitrateAllocation bitrate_allocation; |
| |
| for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { |
| std::vector<DataRate> temporal_layer_rates = |
| SplitBitrate(num_layers_.temporal, spatial_layer_rates[sl_idx], |
| kTemporalLayeringRateScalingFactor); |
| |
| // Distribute rate across temporal layers. Allocate more bits to lower |
| // layers since they are used for prediction of higher layers and their |
| // references are far apart. |
| if (num_layers_.temporal == 1) { |
| bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, |
| temporal_layer_rates[0].bps()); |
| } else if (num_layers_.temporal == 2) { |
| bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, |
| temporal_layer_rates[1].bps()); |
| bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, |
| temporal_layer_rates[0].bps()); |
| } else { |
| RTC_CHECK_EQ(num_layers_.temporal, 3); |
| // In case of three temporal layers the high layer has two frames and the |
| // middle layer has one frame within GOP (in between two consecutive low |
| // layer frames). Thus high layer requires more bits (comparing pure |
| // bitrate of layer, excluding bitrate of base layers) to keep quality on |
| // par with lower layers. |
| bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, |
| temporal_layer_rates[2].bps()); |
| bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, |
| temporal_layer_rates[0].bps()); |
| bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2, |
| temporal_layer_rates[1].bps()); |
| } |
| } |
| |
| return bitrate_allocation; |
| } |
| |
| // Bit-rate is allocated in such a way, that the highest enabled layer will have |
| // between min and max bitrate, and all others will have exactly target |
| // bit-rate allocated. |
| VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( |
| DataRate total_bitrate, |
| size_t first_active_layer, |
| size_t num_spatial_layers) const { |
| VideoBitrateAllocation bitrate_allocation; |
| |
| if (num_spatial_layers == 0 || |
| total_bitrate < |
| DataRate::KilobitsPerSec( |
| codec_.spatialLayers[first_active_layer].minBitrate)) { |
| // Always enable at least one layer. |
| bitrate_allocation.SetBitrate(first_active_layer, 0, total_bitrate.bps()); |
| return bitrate_allocation; |
| } |
| |
| DataRate allocated_rate = DataRate::Zero(); |
| DataRate top_layer_rate = DataRate::Zero(); |
| size_t sl_idx; |
| for (sl_idx = first_active_layer; |
| sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) { |
| const DataRate min_rate = |
| DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx].minBitrate); |
| const DataRate target_rate = |
| DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx].targetBitrate); |
| |
| if (allocated_rate + min_rate > total_bitrate) { |
| // Use stable rate to determine if layer should be enabled. |
| break; |
| } |
| |
| top_layer_rate = std::min(target_rate, total_bitrate - allocated_rate); |
| bitrate_allocation.SetBitrate(sl_idx, 0, top_layer_rate.bps()); |
| allocated_rate += top_layer_rate; |
| } |
| |
| if (sl_idx > 0 && total_bitrate - allocated_rate > DataRate::Zero()) { |
| // Add leftover to the last allocated layer. |
| top_layer_rate = std::min( |
| top_layer_rate + (total_bitrate - allocated_rate), |
| DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx - 1].maxBitrate)); |
| bitrate_allocation.SetBitrate(sl_idx - 1, 0, top_layer_rate.bps()); |
| } |
| |
| return bitrate_allocation; |
| } |
| |
| size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const { |
| if (cumulative_layer_start_bitrates_.empty()) { |
| return 0; |
| } |
| |
| size_t num_enabled_layers = 0; |
| for (DataRate start_rate : cumulative_layer_start_bitrates_) { |
| // First layer is always enabled. |
| if (num_enabled_layers == 0 || start_rate <= target_rate) { |
| ++num_enabled_layers; |
| } else { |
| break; |
| } |
| } |
| |
| return num_enabled_layers; |
| } |
| |
| DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) { |
| const NumLayers num_layers = GetNumLayers(codec); |
| const ActiveSpatialLayers active_layers = |
| GetActiveSpatialLayers(codec, num_layers.spatial); |
| |
| DataRate max_bitrate = DataRate::Zero(); |
| for (size_t sl_idx = 0; sl_idx < active_layers.num; ++sl_idx) { |
| max_bitrate += DataRate::KilobitsPerSec( |
| codec.spatialLayers[active_layers.first + sl_idx].maxBitrate); |
| } |
| |
| if (codec.maxBitrate != 0) { |
| max_bitrate = |
| std::min(max_bitrate, DataRate::KilobitsPerSec(codec.maxBitrate)); |
| } |
| |
| return max_bitrate; |
| } |
| |
| DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) { |
| auto start_bitrate = GetLayerStartBitrates(codec); |
| if (start_bitrate.empty()) { |
| return DataRate::Zero(); // All layers are deactivated. |
| } |
| |
| return start_bitrate.back(); |
| } |
| |
| absl::InlinedVector<DataRate, kMaxSpatialLayers> |
| SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) { |
| absl::InlinedVector<DataRate, kMaxSpatialLayers> start_bitrates; |
| const NumLayers num_layers = GetNumLayers(codec); |
| const ActiveSpatialLayers active_layers = |
| GetActiveSpatialLayers(codec, num_layers.spatial); |
| DataRate last_rate = DataRate::Zero(); |
| for (size_t i = 1; i <= active_layers.num; ++i) { |
| DataRate layer_toggling_rate = |
| FindLayerTogglingThreshold(codec, active_layers.first, i); |
| start_bitrates.push_back(layer_toggling_rate); |
| RTC_DCHECK_LE(last_rate, layer_toggling_rate); |
| last_rate = layer_toggling_rate; |
| } |
| return start_bitrates; |
| } |
| |
| } // namespace webrtc |
