blob: dd054030f1cc8f4d027457432fbcddbc5ebaf3af [file] [log] [blame]
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include <stdio.h>
#include <algorithm>
#include <cstdint>
#include <numeric>
#include <tuple>
#include <vector>
#include "api/environment/environment.h"
#include "api/units/data_rate.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_codec_type.h"
#include "api/video_codecs/simulcast_stream.h"
#include "api/video_codecs/video_codec.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/rate_control_settings.h"
namespace webrtc {
namespace {
// Ratio allocation between temporal streams:
// Values as required for the VP8 codec (accumulating).
static const float
kLayerRateAllocation[kMaxTemporalStreams][kMaxTemporalStreams] = {
{1.0f, 1.0f, 1.0f, 1.0f}, // 1 layer
{0.6f, 1.0f, 1.0f, 1.0f}, // 2 layers {60%, 40%}
{0.4f, 0.6f, 1.0f, 1.0f}, // 3 layers {40%, 20%, 40%}
{0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%}
};
static const float kBaseHeavy3TlRateAllocation[kMaxTemporalStreams] = {
0.6f, 0.8f, 1.0f, 1.0f // 3 layers {60%, 20%, 20%}
};
const uint32_t kLegacyScreenshareTl0BitrateKbps = 200;
const uint32_t kLegacyScreenshareTl1BitrateKbps = 1000;
} // namespace
float SimulcastRateAllocator::GetTemporalRateAllocation(
int num_layers,
int temporal_id,
bool base_heavy_tl3_alloc) {
RTC_CHECK_GT(num_layers, 0);
RTC_CHECK_LE(num_layers, kMaxTemporalStreams);
RTC_CHECK_GE(temporal_id, 0);
RTC_CHECK_LT(temporal_id, num_layers);
if (num_layers == 3 && base_heavy_tl3_alloc) {
return kBaseHeavy3TlRateAllocation[temporal_id];
}
return kLayerRateAllocation[num_layers - 1][temporal_id];
}
SimulcastRateAllocator::SimulcastRateAllocator(const Environment& env,
const VideoCodec& codec)
: codec_(codec),
stable_rate_settings_(env.field_trials()),
rate_control_settings_(env.field_trials()),
legacy_conference_mode_(false) {}
SimulcastRateAllocator::~SimulcastRateAllocator() = default;
VideoBitrateAllocation SimulcastRateAllocator::Allocate(
VideoBitrateAllocationParameters parameters) {
VideoBitrateAllocation allocated_bitrates;
DataRate stable_rate = parameters.total_bitrate;
if (stable_rate_settings_.IsEnabled() &&
parameters.stable_bitrate > DataRate::Zero()) {
stable_rate = std::min(parameters.stable_bitrate, parameters.total_bitrate);
}
DistributeAllocationToSimulcastLayers(parameters.total_bitrate, stable_rate,
&allocated_bitrates);
DistributeAllocationToTemporalLayers(&allocated_bitrates);
return allocated_bitrates;
}
void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers(
DataRate total_bitrate,
DataRate stable_bitrate,
VideoBitrateAllocation* allocated_bitrates) {
DataRate left_in_total_allocation = total_bitrate;
DataRate left_in_stable_allocation = stable_bitrate;
if (codec_.maxBitrate) {
DataRate max_rate = DataRate::KilobitsPerSec(codec_.maxBitrate);
left_in_total_allocation = std::min(left_in_total_allocation, max_rate);
left_in_stable_allocation = std::min(left_in_stable_allocation, max_rate);
}
if (codec_.numberOfSimulcastStreams == 0) {
// No simulcast, just set the target as this has been capped already.
if (codec_.active) {
allocated_bitrates->SetBitrate(
0, 0,
std::max(DataRate::KilobitsPerSec(codec_.minBitrate),
left_in_total_allocation)
.bps());
}
return;
}
// Sort the layers by maxFramerate, they might not always be from smallest
// to biggest
std::vector<size_t> layer_index(codec_.numberOfSimulcastStreams);
std::iota(layer_index.begin(), layer_index.end(), 0);
std::stable_sort(layer_index.begin(), layer_index.end(),
[this](size_t a, size_t b) {
return std::tie(codec_.simulcastStream[a].maxBitrate) <
std::tie(codec_.simulcastStream[b].maxBitrate);
});
// Find the first active layer. We don't allocate to inactive layers.
size_t active_layer = 0;
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
if (codec_.simulcastStream[layer_index[active_layer]].active) {
// Found the first active layer.
break;
}
}
// All streams could be inactive, and nothing more to do.
if (active_layer == codec_.numberOfSimulcastStreams) {
return;
}
// Always allocate enough bitrate for the minimum bitrate of the first
// active layer. Suspending below min bitrate is controlled outside the
// codec implementation and is not overridden by this.
DataRate min_rate = DataRate::KilobitsPerSec(
codec_.simulcastStream[layer_index[active_layer]].minBitrate);
left_in_total_allocation = std::max(left_in_total_allocation, min_rate);
left_in_stable_allocation = std::max(left_in_stable_allocation, min_rate);
// Begin by allocating bitrate to simulcast streams, putting all bitrate in
// temporal layer 0. We'll then distribute this bitrate, across potential
// temporal layers, when stream allocation is done.
bool first_allocation = false;
if (stream_enabled_.empty()) {
// First time allocating, this means we should not include hysteresis in
// case this is a reconfiguration of an existing enabled stream.
first_allocation = true;
stream_enabled_.resize(codec_.numberOfSimulcastStreams, false);
}
size_t top_active_layer = active_layer;
// Allocate up to the target bitrate for each active simulcast layer.
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
const SimulcastStream& stream =
codec_.simulcastStream[layer_index[active_layer]];
if (!stream.active) {
stream_enabled_[layer_index[active_layer]] = false;
continue;
}
// If we can't allocate to the current layer we can't allocate to higher
// layers because they require a higher minimum bitrate.
DataRate min_bitrate = DataRate::KilobitsPerSec(stream.minBitrate);
DataRate target_bitrate = DataRate::KilobitsPerSec(stream.targetBitrate);
double hysteresis_factor =
codec_.mode == VideoCodecMode::kRealtimeVideo
? stable_rate_settings_.GetVideoHysteresisFactor()
: stable_rate_settings_.GetScreenshareHysteresisFactor();
if (!first_allocation && !stream_enabled_[layer_index[active_layer]]) {
min_bitrate = std::min(hysteresis_factor * min_bitrate, target_bitrate);
}
if (left_in_stable_allocation < min_bitrate) {
allocated_bitrates->set_bw_limited(true);
break;
}
// We are allocating to this layer so it is the current active allocation.
top_active_layer = layer_index[active_layer];
stream_enabled_[layer_index[active_layer]] = true;
DataRate layer_rate = std::min(left_in_total_allocation, target_bitrate);
allocated_bitrates->SetBitrate(layer_index[active_layer], 0,
layer_rate.bps());
left_in_total_allocation -= layer_rate;
left_in_stable_allocation -=
std::min(left_in_stable_allocation, target_bitrate);
}
// All layers above this one are not active.
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
stream_enabled_[layer_index[active_layer]] = false;
}
// Next, try allocate remaining bitrate, up to max bitrate, in top active
// stream.
// TODO(sprang): Allocate up to max bitrate for all layers once we have a
// better idea of possible performance implications.
if (left_in_total_allocation > DataRate::Zero()) {
const SimulcastStream& stream = codec_.simulcastStream[top_active_layer];
DataRate initial_layer_rate = DataRate::BitsPerSec(
allocated_bitrates->GetSpatialLayerSum(top_active_layer));
DataRate additional_allocation = std::min(
left_in_total_allocation,
DataRate::KilobitsPerSec(stream.maxBitrate) - initial_layer_rate);
allocated_bitrates->SetBitrate(
top_active_layer, 0,
(initial_layer_rate + additional_allocation).bps());
}
}
void SimulcastRateAllocator::DistributeAllocationToTemporalLayers(
VideoBitrateAllocation* allocated_bitrates_bps) const {
const int num_spatial_streams =
std::max(1, static_cast<int>(codec_.numberOfSimulcastStreams));
// Finally, distribute the bitrate for the simulcast streams across the
// available temporal layers.
for (int simulcast_id = 0; simulcast_id < num_spatial_streams;
++simulcast_id) {
uint32_t target_bitrate_kbps =
allocated_bitrates_bps->GetBitrate(simulcast_id, 0) / 1000;
if (target_bitrate_kbps == 0) {
continue;
}
const uint32_t expected_allocated_bitrate_kbps = target_bitrate_kbps;
RTC_DCHECK_EQ(
target_bitrate_kbps,
allocated_bitrates_bps->GetSpatialLayerSum(simulcast_id) / 1000);
const int num_temporal_streams = NumTemporalStreams(simulcast_id);
uint32_t max_bitrate_kbps;
// Legacy temporal-layered only screenshare, or simulcast screenshare
// with legacy mode for simulcast stream 0.
if (codec_.mode == VideoCodecMode::kScreensharing &&
legacy_conference_mode_ && simulcast_id == 0) {
// TODO(holmer): This is a "temporary" hack for screensharing, where we
// interpret the startBitrate as the encoder target bitrate. This is
// to allow for a different max bitrate, so if the codec can't meet
// the target we still allow it to overshoot up to the max before dropping
// frames. This hack should be improved.
max_bitrate_kbps =
std::min(kLegacyScreenshareTl1BitrateKbps, target_bitrate_kbps);
target_bitrate_kbps =
std::min(kLegacyScreenshareTl0BitrateKbps, target_bitrate_kbps);
} else if (num_spatial_streams == 1) {
max_bitrate_kbps = codec_.maxBitrate;
} else {
max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate;
}
std::vector<uint32_t> tl_allocation;
if (num_temporal_streams == 1) {
tl_allocation.push_back(target_bitrate_kbps);
} else {
if (codec_.mode == VideoCodecMode::kScreensharing &&
legacy_conference_mode_ && simulcast_id == 0) {
tl_allocation = ScreenshareTemporalLayerAllocation(
target_bitrate_kbps, max_bitrate_kbps, simulcast_id);
} else {
tl_allocation = DefaultTemporalLayerAllocation(
target_bitrate_kbps, max_bitrate_kbps, simulcast_id);
}
}
RTC_DCHECK_GT(tl_allocation.size(), 0);
RTC_DCHECK_LE(tl_allocation.size(), num_temporal_streams);
uint64_t tl_allocation_sum_kbps = 0;
for (size_t tl_index = 0; tl_index < tl_allocation.size(); ++tl_index) {
uint32_t layer_rate_kbps = tl_allocation[tl_index];
if (layer_rate_kbps > 0) {
allocated_bitrates_bps->SetBitrate(simulcast_id, tl_index,
layer_rate_kbps * 1000);
}
tl_allocation_sum_kbps += layer_rate_kbps;
}
RTC_DCHECK_LE(tl_allocation_sum_kbps, expected_allocated_bitrate_kbps);
}
}
std::vector<uint32_t> SimulcastRateAllocator::DefaultTemporalLayerAllocation(
int bitrate_kbps,
int max_bitrate_kbps,
int simulcast_id) const {
const size_t num_temporal_layers = NumTemporalStreams(simulcast_id);
std::vector<uint32_t> bitrates;
for (size_t i = 0; i < num_temporal_layers; ++i) {
float layer_bitrate =
bitrate_kbps *
GetTemporalRateAllocation(
num_temporal_layers, i,
rate_control_settings_.Vp8BaseHeavyTl3RateAllocation());
bitrates.push_back(static_cast<uint32_t>(layer_bitrate + 0.5));
}
// Allocation table is of aggregates, transform to individual rates.
uint32_t sum = 0;
for (size_t i = 0; i < num_temporal_layers; ++i) {
uint32_t layer_bitrate = bitrates[i];
RTC_DCHECK_LE(sum, bitrates[i]);
bitrates[i] -= sum;
sum = layer_bitrate;
if (sum >= static_cast<uint32_t>(bitrate_kbps)) {
// Sum adds up; any subsequent layers will be 0.
bitrates.resize(i + 1);
break;
}
}
return bitrates;
}
std::vector<uint32_t>
SimulcastRateAllocator::ScreenshareTemporalLayerAllocation(
int bitrate_kbps,
int max_bitrate_kbps,
int simulcast_id) const {
if (simulcast_id > 0) {
return DefaultTemporalLayerAllocation(bitrate_kbps, max_bitrate_kbps,
simulcast_id);
}
std::vector<uint32_t> allocation;
allocation.push_back(bitrate_kbps);
if (max_bitrate_kbps > bitrate_kbps)
allocation.push_back(max_bitrate_kbps - bitrate_kbps);
return allocation;
}
const VideoCodec& webrtc::SimulcastRateAllocator::GetCodec() const {
return codec_;
}
int SimulcastRateAllocator::NumTemporalStreams(size_t simulcast_id) const {
return std::max<uint8_t>(
1,
codec_.codecType == kVideoCodecVP8 && codec_.numberOfSimulcastStreams == 0
? codec_.VP8().numberOfTemporalLayers
: codec_.simulcastStream[simulcast_id].numberOfTemporalLayers);
}
void SimulcastRateAllocator::SetLegacyConferenceMode(bool enabled) {
legacy_conference_mode_ = enabled;
}
} // namespace webrtc