modules/video_coding/utility/simulcast_rate_allocator.cc

/*
 *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/video_coding/utility/simulcast_rate_allocator.h"

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <numeric>
#include <tuple>
#include <vector>

#include "api/environment/environment.h"
#include "api/units/data_rate.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_codec_type.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/simulcast_stream.h"
#include "api/video_codecs/video_codec.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/rate_control_settings.h"

namespace webrtc {
namespace {
// Ratio allocation between temporal streams:
// Values as required for the VP8 codec (accumulating).
const float kLayerRateAllocation[kMaxTemporalStreams][kMaxTemporalStreams] = {
    {1.0f, 1.0f, 1.0f, 1.0f},  // 1 layer
    {0.6f, 1.0f, 1.0f, 1.0f},  // 2 layers {60%, 40%}
    {0.4f, 0.6f, 1.0f, 1.0f},  // 3 layers {40%, 20%, 40%}
    {0.25f, 0.4f, 0.6f, 1.0f}  // 4 layers {25%, 15%, 20%, 40%}
};

const float kBaseHeavy3TlRateAllocation[kMaxTemporalStreams] = {
    0.6f, 0.8f, 1.0f, 1.0f  // 3 layers {60%, 20%, 20%}
};

const uint32_t kLegacyScreenshareTl0BitrateKbps = 200;
const uint32_t kLegacyScreenshareTl1BitrateKbps = 1000;

constexpr double kVideoHysteresisFactor = 1.2;
constexpr double kScreenshareHysteresisFactor = 1.35;
}  // namespace

float SimulcastRateAllocator::GetTemporalRateAllocation(
    int num_layers,
    int temporal_id,
    bool base_heavy_tl3_alloc) {
  RTC_CHECK_GT(num_layers, 0);
  RTC_CHECK_LE(num_layers, kMaxTemporalStreams);
  RTC_CHECK_GE(temporal_id, 0);
  RTC_CHECK_LT(temporal_id, num_layers);
  if (num_layers == 3 && base_heavy_tl3_alloc) {
    return kBaseHeavy3TlRateAllocation[temporal_id];
  }
  return kLayerRateAllocation[num_layers - 1][temporal_id];
}

SimulcastRateAllocator::SimulcastRateAllocator(const Environment& env,
                                               const VideoCodec& codec)
    : codec_(codec),
      rate_control_settings_(env.field_trials()),
      legacy_conference_mode_(false) {}

SimulcastRateAllocator::~SimulcastRateAllocator() = default;

VideoBitrateAllocation SimulcastRateAllocator::Allocate(
    VideoBitrateAllocationParameters parameters) {
  VideoBitrateAllocation allocated_bitrates;
  DataRate stable_rate = parameters.total_bitrate;
  DistributeAllocationToSimulcastLayers(parameters.total_bitrate, stable_rate,
                                        &allocated_bitrates);
  DistributeAllocationToTemporalLayers(&allocated_bitrates);
  return allocated_bitrates;
}

void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers(
    DataRate total_bitrate,
    DataRate stable_bitrate,
    VideoBitrateAllocation* allocated_bitrates) {
  DataRate left_in_total_allocation = total_bitrate;
  DataRate left_in_stable_allocation = stable_bitrate;

  if (codec_.maxBitrate) {
    DataRate max_rate = DataRate::KilobitsPerSec(codec_.maxBitrate);
    left_in_total_allocation = std::min(left_in_total_allocation, max_rate);
    left_in_stable_allocation = std::min(left_in_stable_allocation, max_rate);
  }

  if (codec_.numberOfSimulcastStreams == 0) {
    // No simulcast, just set the target as this has been capped already.
    if (codec_.active) {
      allocated_bitrates->SetBitrate(
          0, 0,
          std::max(DataRate::KilobitsPerSec(codec_.minBitrate),
                   left_in_total_allocation)
              .bps());
    }
    return;
  }

  // Sort the layers by maxFramerate, they might not always be from smallest
  // to biggest
  std::vector<size_t> layer_index(codec_.numberOfSimulcastStreams);
  std::iota(layer_index.begin(), layer_index.end(), 0);
  std::stable_sort(layer_index.begin(), layer_index.end(),
                   [this](size_t a, size_t b) {
                     return std::tie(codec_.simulcastStream[a].maxBitrate) <
                            std::tie(codec_.simulcastStream[b].maxBitrate);
                   });

  // Find the first active layer. We don't allocate to inactive layers.
  size_t active_layer = 0;
  for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
    if (codec_.simulcastStream[layer_index[active_layer]].active) {
      // Found the first active layer.
      break;
    }
  }
  // All streams could be inactive, and nothing more to do.
  if (active_layer == codec_.numberOfSimulcastStreams) {
    return;
  }

  // Always allocate enough bitrate for the minimum bitrate of the first
  // active layer. Suspending below min bitrate is controlled outside the
  // codec implementation and is not overridden by this.
  DataRate min_rate = DataRate::KilobitsPerSec(
      codec_.simulcastStream[layer_index[active_layer]].minBitrate);
  left_in_total_allocation = std::max(left_in_total_allocation, min_rate);
  left_in_stable_allocation = std::max(left_in_stable_allocation, min_rate);

  // Begin by allocating bitrate to simulcast streams, putting all bitrate in
  // temporal layer 0. We'll then distribute this bitrate, across potential
  // temporal layers, when stream allocation is done.

  bool first_allocation = false;
  if (stream_enabled_.empty()) {
    // First time allocating, this means we should not include hysteresis in
    // case this is a reconfiguration of an existing enabled stream.
    first_allocation = true;
    stream_enabled_.resize(codec_.numberOfSimulcastStreams, false);
  }

  size_t top_active_layer = active_layer;
  // Allocate up to the target bitrate for each active simulcast layer.
  for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
    const SimulcastStream& stream =
        codec_.simulcastStream[layer_index[active_layer]];
    if (!stream.active) {
      stream_enabled_[layer_index[active_layer]] = false;
      continue;
    }
    // If we can't allocate to the current layer we can't allocate to higher
    // layers because they require a higher minimum bitrate.
    DataRate min_bitrate = DataRate::KilobitsPerSec(stream.minBitrate);
    DataRate target_bitrate = DataRate::KilobitsPerSec(stream.targetBitrate);
    double hysteresis_factor = codec_.mode == VideoCodecMode::kRealtimeVideo
                                   ? kVideoHysteresisFactor
                                   : kScreenshareHysteresisFactor;
    if (!first_allocation && !stream_enabled_[layer_index[active_layer]]) {
      min_bitrate = std::min(hysteresis_factor * min_bitrate, target_bitrate);
    }
    if (left_in_stable_allocation < min_bitrate) {
      allocated_bitrates->set_bw_limited(true);
      break;
    }

    // We are allocating to this layer so it is the current active allocation.
    top_active_layer = layer_index[active_layer];
    stream_enabled_[layer_index[active_layer]] = true;
    DataRate layer_rate = std::min(left_in_total_allocation, target_bitrate);
    allocated_bitrates->SetBitrate(layer_index[active_layer], 0,
                                   layer_rate.bps());
    left_in_total_allocation -= layer_rate;
    left_in_stable_allocation -=
        std::min(left_in_stable_allocation, target_bitrate);
  }

  // All layers above this one are not active.
  for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
    stream_enabled_[layer_index[active_layer]] = false;
  }

  // Next, try allocate remaining bitrate, up to max bitrate, in top active
  // stream.
  // TODO(sprang): Allocate up to max bitrate for all layers once we have a
  //               better idea of possible performance implications.
  if (left_in_total_allocation > DataRate::Zero()) {
    const SimulcastStream& stream = codec_.simulcastStream[top_active_layer];
    DataRate initial_layer_rate = DataRate::BitsPerSec(
        allocated_bitrates->GetSpatialLayerSum(top_active_layer));
    DataRate additional_allocation = std::min(
        left_in_total_allocation,
        DataRate::KilobitsPerSec(stream.maxBitrate) - initial_layer_rate);
    allocated_bitrates->SetBitrate(
        top_active_layer, 0,
        (initial_layer_rate + additional_allocation).bps());
  }
}

void SimulcastRateAllocator::DistributeAllocationToTemporalLayers(
    VideoBitrateAllocation* allocated_bitrates_bps) const {
  const int num_spatial_streams =
      std::max(1, static_cast<int>(codec_.numberOfSimulcastStreams));

  // Finally, distribute the bitrate for the simulcast streams across the
  // available temporal layers.
  for (int simulcast_id = 0; simulcast_id < num_spatial_streams;
       ++simulcast_id) {
    uint32_t target_bitrate_kbps =
        allocated_bitrates_bps->GetBitrate(simulcast_id, 0) / 1000;
    if (target_bitrate_kbps == 0) {
      continue;
    }

    const uint32_t expected_allocated_bitrate_kbps = target_bitrate_kbps;
    RTC_DCHECK_EQ(
        target_bitrate_kbps,
        allocated_bitrates_bps->GetSpatialLayerSum(simulcast_id) / 1000);
    const int num_temporal_streams = NumTemporalStreams(simulcast_id);
    uint32_t max_bitrate_kbps;
    // Legacy temporal-layered only screenshare, or simulcast screenshare
    // with legacy mode for simulcast stream 0.
    if (codec_.mode == VideoCodecMode::kScreensharing &&
        legacy_conference_mode_ && simulcast_id == 0) {
      // TODO(holmer): This is a "temporary" hack for screensharing, where we
      // interpret the startBitrate as the encoder target bitrate. This is
      // to allow for a different max bitrate, so if the codec can't meet
      // the target we still allow it to overshoot up to the max before dropping
      // frames. This hack should be improved.
      max_bitrate_kbps =
          std::min(kLegacyScreenshareTl1BitrateKbps, target_bitrate_kbps);
      target_bitrate_kbps =
          std::min(kLegacyScreenshareTl0BitrateKbps, target_bitrate_kbps);
    } else if (num_spatial_streams == 1) {
      max_bitrate_kbps = codec_.maxBitrate;
    } else {
      max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate;
    }

    std::vector<uint32_t> tl_allocation;
    if (num_temporal_streams == 1) {
      tl_allocation.push_back(target_bitrate_kbps);
    } else {
      if (codec_.mode == VideoCodecMode::kScreensharing &&
          legacy_conference_mode_ && simulcast_id == 0) {
        tl_allocation = ScreenshareTemporalLayerAllocation(
            target_bitrate_kbps, max_bitrate_kbps, simulcast_id);
      } else {
        tl_allocation = DefaultTemporalLayerAllocation(
            target_bitrate_kbps, max_bitrate_kbps, simulcast_id);
      }
    }
    RTC_DCHECK_GT(tl_allocation.size(), 0);
    RTC_DCHECK_LE(tl_allocation.size(), num_temporal_streams);

    uint64_t tl_allocation_sum_kbps = 0;
    for (size_t tl_index = 0; tl_index < tl_allocation.size(); ++tl_index) {
      uint32_t layer_rate_kbps = tl_allocation[tl_index];
      if (layer_rate_kbps > 0) {
        allocated_bitrates_bps->SetBitrate(simulcast_id, tl_index,
                                           layer_rate_kbps * 1000);
      }
      tl_allocation_sum_kbps += layer_rate_kbps;
    }
    RTC_DCHECK_LE(tl_allocation_sum_kbps, expected_allocated_bitrate_kbps);
  }
}

std::vector<uint32_t> SimulcastRateAllocator::DefaultTemporalLayerAllocation(
    int bitrate_kbps,
    int /* max_bitrate_kbps */,
    int simulcast_id) const {
  const size_t num_temporal_layers = NumTemporalStreams(simulcast_id);
  std::vector<uint32_t> bitrates;
  for (size_t i = 0; i < num_temporal_layers; ++i) {
    float layer_bitrate =
        bitrate_kbps *
        GetTemporalRateAllocation(
            num_temporal_layers, i,
            rate_control_settings_.Vp8BaseHeavyTl3RateAllocation());
    bitrates.push_back(static_cast<uint32_t>(layer_bitrate + 0.5));
  }

  // Allocation table is of aggregates, transform to individual rates.
  uint32_t sum = 0;
  for (size_t i = 0; i < num_temporal_layers; ++i) {
    uint32_t layer_bitrate = bitrates[i];
    RTC_DCHECK_LE(sum, bitrates[i]);
    bitrates[i] -= sum;
    sum = layer_bitrate;

    if (sum >= static_cast<uint32_t>(bitrate_kbps)) {
      // Sum adds up; any subsequent layers will be 0.
      bitrates.resize(i + 1);
      break;
    }
  }

  return bitrates;
}

std::vector<uint32_t>
SimulcastRateAllocator::ScreenshareTemporalLayerAllocation(
    int bitrate_kbps,
    int max_bitrate_kbps,
    int simulcast_id) const {
  if (simulcast_id > 0) {
    return DefaultTemporalLayerAllocation(bitrate_kbps, max_bitrate_kbps,
                                          simulcast_id);
  }
  std::vector<uint32_t> allocation;
  allocation.push_back(bitrate_kbps);
  if (max_bitrate_kbps > bitrate_kbps)
    allocation.push_back(max_bitrate_kbps - bitrate_kbps);
  return allocation;
}

const VideoCodec& SimulcastRateAllocator::GetCodec() const {
  return codec_;
}

int SimulcastRateAllocator::NumTemporalStreams(size_t simulcast_id) const {
  bool is_vp8 = codec_.simulcastStream[simulcast_id].format
                    ? codec_.simulcastStream[simulcast_id].format->IsSameCodec(
                          webrtc::SdpVideoFormat::VP8())
                    : codec_.codecType == kVideoCodecVP8;
  return std::max<uint8_t>(
      1, is_vp8 && codec_.numberOfSimulcastStreams == 0
             ? codec_.VP8().numberOfTemporalLayers
             : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers);
}

void SimulcastRateAllocator::SetLegacyConferenceMode(bool enabled) {
  legacy_conference_mode_ = enabled;
}

}  // namespace webrtc