media/engine/simulcast_encoder_adapter.cc

/*
 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "media/engine/simulcast_encoder_adapter.h"

#include <stdio.h>
#include <string.h>

#include <algorithm>
#include <cstdint>
#include <iterator>
#include <memory>
#include <numeric>
#include <optional>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

#include "absl/algorithm/container.h"
#include "absl/base/nullability.h"
#include "api/array_view.h"
#include "api/environment/environment.h"
#include "api/fec_controller_override.h"
#include "api/field_trials_view.h"
#include "api/scoped_refptr.h"
#include "api/sequence_checker.h"
#include "api/units/data_rate.h"
#include "api/units/timestamp.h"
#include "api/video/encoded_image.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_codec_type.h"
#include "api/video/video_frame.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_frame_type.h"
#include "api/video/video_rotation.h"
#include "api/video_codecs/scalability_mode.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/simulcast_stream.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
#include "common_video/framerate_controller.h"
#include "media/base/sdp_video_format_utils.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/include/video_error_codes_utils.h"
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/str_join.h"
#include "rtc_base/strings/string_builder.h"

namespace webrtc {
namespace {

// Max qp for lowest spatial resolution when doing simulcast.
const unsigned int kLowestResMaxQp = 45;

uint32_t SumStreamMaxBitrate(int streams, const VideoCodec& codec) {
  uint32_t bitrate_sum = 0;
  for (int i = 0; i < streams; ++i) {
    bitrate_sum += codec.simulcastStream[i].maxBitrate;
  }
  return bitrate_sum;
}

int CountAllStreams(const VideoCodec& codec) {
  int total_streams_count =
      codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
  uint32_t simulcast_max_bitrate =
      SumStreamMaxBitrate(total_streams_count, codec);
  if (simulcast_max_bitrate == 0) {
    total_streams_count = 1;
  }
  return total_streams_count;
}

int CountActiveStreams(const VideoCodec& codec) {
  if (codec.numberOfSimulcastStreams < 1) {
    return 1;
  }
  int total_streams_count = CountAllStreams(codec);
  int active_streams_count = 0;
  for (int i = 0; i < total_streams_count; ++i) {
    if (codec.simulcastStream[i].active) {
      ++active_streams_count;
    }
  }
  return active_streams_count;
}

int VerifyCodec(const VideoCodec* codec_settings) {
  if (codec_settings == nullptr) {
    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  }
  if (codec_settings->maxFramerate < 1) {
    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  }
  // allow zero to represent an unspecified maxBitRate
  if (codec_settings->maxBitrate > 0 &&
      codec_settings->startBitrate > codec_settings->maxBitrate) {
    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  }
  if (codec_settings->width <= 1 || codec_settings->height <= 1) {
    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  }
  if (codec_settings->codecType == webrtc::kVideoCodecVP8 &&
      codec_settings->VP8().automaticResizeOn &&
      CountActiveStreams(*codec_settings) > 1) {
    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  }
  return WEBRTC_VIDEO_CODEC_OK;
}

bool StreamQualityCompare(const SimulcastStream& a, const SimulcastStream& b) {
  return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) <
         std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate);
}

void GetLowestAndHighestQualityStreamIndixes(
    rtc::ArrayView<const SimulcastStream> streams,
    int* lowest_quality_stream_idx,
    int* highest_quality_stream_idx) {
  const auto lowest_highest_quality_streams =
      absl::c_minmax_element(streams, StreamQualityCompare);
  *lowest_quality_stream_idx =
      std::distance(streams.begin(), lowest_highest_quality_streams.first);
  *highest_quality_stream_idx =
      std::distance(streams.begin(), lowest_highest_quality_streams.second);
}

std::vector<uint32_t> GetStreamStartBitratesKbps(const Environment& env,
                                                 const VideoCodec& codec) {
  std::vector<uint32_t> start_bitrates;
  VideoBitrateAllocation allocation =
      SimulcastRateAllocator(env, codec)
          .Allocate(VideoBitrateAllocationParameters(codec.startBitrate * 1000,
                                                     codec.maxFramerate));

  int total_streams_count = CountAllStreams(codec);
  for (int i = 0; i < total_streams_count; ++i) {
    uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000;
    start_bitrates.push_back(stream_bitrate);
  }
  return start_bitrates;
}

}  // namespace

SimulcastEncoderAdapter::EncoderContext::EncoderContext(
    std::unique_ptr<VideoEncoder> encoder,
    bool prefer_temporal_support,
    VideoEncoder::EncoderInfo primary_info,
    VideoEncoder::EncoderInfo fallback_info)
    : encoder_(std::move(encoder)),
      prefer_temporal_support_(prefer_temporal_support),
      primary_info_(std::move(primary_info)),
      fallback_info_(std::move(fallback_info)) {}

void SimulcastEncoderAdapter::EncoderContext::Release() {
  if (encoder_) {
    encoder_->Release();
    encoder_->RegisterEncodeCompleteCallback(nullptr);
  }
}

SimulcastEncoderAdapter::StreamContext::StreamContext(
    SimulcastEncoderAdapter* parent,
    std::unique_ptr<EncoderContext> encoder_context,
    std::unique_ptr<FramerateController> framerate_controller,
    int stream_idx,
    uint16_t width,
    uint16_t height,
    bool is_paused)
    : parent_(parent),
      encoder_context_(std::move(encoder_context)),
      framerate_controller_(std::move(framerate_controller)),
      stream_idx_(stream_idx),
      width_(width),
      height_(height),
      is_keyframe_needed_(false),
      is_paused_(is_paused) {
  if (parent_) {
    encoder_context_->encoder().RegisterEncodeCompleteCallback(this);
  }
}

SimulcastEncoderAdapter::StreamContext::StreamContext(StreamContext&& rhs)
    : parent_(rhs.parent_),
      encoder_context_(std::move(rhs.encoder_context_)),
      framerate_controller_(std::move(rhs.framerate_controller_)),
      stream_idx_(rhs.stream_idx_),
      width_(rhs.width_),
      height_(rhs.height_),
      is_keyframe_needed_(rhs.is_keyframe_needed_),
      is_paused_(rhs.is_paused_) {
  if (parent_) {
    encoder_context_->encoder().RegisterEncodeCompleteCallback(this);
  }
}

SimulcastEncoderAdapter::StreamContext::~StreamContext() {
  if (encoder_context_) {
    encoder_context_->Release();
  }
}

std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
SimulcastEncoderAdapter::StreamContext::ReleaseEncoderContext() && {
  encoder_context_->Release();
  return std::move(encoder_context_);
}

void SimulcastEncoderAdapter::StreamContext::OnKeyframe(Timestamp timestamp) {
  is_keyframe_needed_ = false;
  if (framerate_controller_) {
    framerate_controller_->KeepFrame(timestamp.us() * 1000);
  }
}

bool SimulcastEncoderAdapter::StreamContext::ShouldDropFrame(
    Timestamp timestamp) {
  if (!framerate_controller_) {
    return false;
  }
  return framerate_controller_->ShouldDropFrame(timestamp.us() * 1000);
}

EncodedImageCallback::Result
SimulcastEncoderAdapter::StreamContext::OnEncodedImage(
    const EncodedImage& encoded_image,
    const CodecSpecificInfo* codec_specific_info) {
  RTC_CHECK(parent_);  // If null, this method should never be called.
  return parent_->OnEncodedImage(stream_idx_, encoded_image,
                                 codec_specific_info);
}

void SimulcastEncoderAdapter::StreamContext::OnDroppedFrame(
    DropReason /*reason*/) {
  RTC_CHECK(parent_);  // If null, this method should never be called.
  parent_->OnDroppedFrame(stream_idx_);
}

SimulcastEncoderAdapter::SimulcastEncoderAdapter(
    const Environment& env,
    absl::Nonnull<VideoEncoderFactory*> primary_factory,
    absl::Nullable<VideoEncoderFactory*> fallback_factory,
    const SdpVideoFormat& format)
    : env_(env),
      inited_(0),
      primary_encoder_factory_(primary_factory),
      fallback_encoder_factory_(fallback_factory),
      video_format_(format),
      total_streams_count_(0),
      bypass_mode_(false),
      encoded_complete_callback_(nullptr),
      boost_base_layer_quality_(
          RateControlSettings(env_.field_trials()).Vp8BoostBaseLayerQuality()),
      prefer_temporal_support_on_base_layer_(env_.field_trials().IsEnabled(
          "WebRTC-Video-PreferTemporalSupportOnBaseLayer")),
      per_layer_pli_(SupportsPerLayerPictureLossIndication(format.parameters)),
      encoder_info_override_(env.field_trials()) {
  RTC_DCHECK(primary_factory);

  // The adapter is typically created on the worker thread, but operated on
  // the encoder task queue.
  encoder_queue_.Detach();
}

SimulcastEncoderAdapter::~SimulcastEncoderAdapter() {
  RTC_DCHECK_RUN_ON(&encoder_queue_);
  RTC_DCHECK(!Initialized());
  DestroyStoredEncoders();
}

void SimulcastEncoderAdapter::SetFecControllerOverride(
    FecControllerOverride* /*fec_controller_override*/) {
  // Ignored.
}

int SimulcastEncoderAdapter::Release() {
  RTC_DCHECK_RUN_ON(&encoder_queue_);

  while (!stream_contexts_.empty()) {
    // Move the encoder instances and put it on the `cached_encoder_contexts_`
    // where it may possibly be reused from (ordering does not matter).
    cached_encoder_contexts_.push_front(
        std::move(stream_contexts_.back()).ReleaseEncoderContext());
    stream_contexts_.pop_back();
  }

  bypass_mode_ = false;

  // It's legal to move the encoder to another queue now.
  encoder_queue_.Detach();

  inited_.store(0);

  return WEBRTC_VIDEO_CODEC_OK;
}

int SimulcastEncoderAdapter::InitEncode(
    const VideoCodec* codec_settings,
    const VideoEncoder::Settings& settings) {
  RTC_DCHECK_RUN_ON(&encoder_queue_);

  if (settings.number_of_cores < 1) {
    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  }

  int ret = VerifyCodec(codec_settings);
  if (ret < 0) {
    return ret;
  }

  Release();

  codec_ = *codec_settings;
  total_streams_count_ = CountAllStreams(*codec_settings);

  bool is_legacy_singlecast = codec_.numberOfSimulcastStreams == 0;
  int lowest_quality_stream_idx = 0;
  int highest_quality_stream_idx = 0;
  if (!is_legacy_singlecast) {
    GetLowestAndHighestQualityStreamIndixes(
        rtc::ArrayView<SimulcastStream>(codec_.simulcastStream,
                                        total_streams_count_),
        &lowest_quality_stream_idx, &highest_quality_stream_idx);
  }

  std::unique_ptr<EncoderContext> encoder_context = FetchOrCreateEncoderContext(
      /*is_lowest_quality_stream=*/(
          is_legacy_singlecast ||
          codec_.simulcastStream[lowest_quality_stream_idx].active));
  if (encoder_context == nullptr) {
    return WEBRTC_VIDEO_CODEC_MEMORY;
  }

  // Two distinct scenarios:
  // * Singlecast (total_streams_count == 1) or simulcast with simulcast-capable
  //   underlaying encoder implementation if active_streams_count > 1. SEA
  //   operates in bypass mode: original settings are passed to the underlaying
  //   encoder, frame encode complete callback is not intercepted.
  // * Multi-encoder simulcast or singlecast if layers are deactivated
  //   (active_streams_count >= 1). SEA creates N=active_streams_count encoders
  //   and configures each to produce a single stream.

  int active_streams_count = CountActiveStreams(*codec_settings);
  // If we only have a single active layer it is better to create an encoder
  // with only one configured layer than creating it with all-but-one disabled
  // layers because that way we control scaling.
  // The use of the nonstandard x-google-per-layer-pli fmtp parameter also
  // forces the use of SEA with separate encoders to support per-layer
  // handling of PLIs.
  bool separate_encoders_needed =
      !encoder_context->encoder().GetEncoderInfo().supports_simulcast ||
      active_streams_count == 1 || per_layer_pli_;
  RTC_LOG(LS_INFO) << "[SEA] InitEncode: total_streams_count: "
                   << total_streams_count_
                   << ", active_streams_count: " << active_streams_count
                   << ", separate_encoders_needed: "
                   << (separate_encoders_needed ? "true" : "false");
  // Singlecast or simulcast with simulcast-capable underlaying encoder.
  if (total_streams_count_ == 1 || !separate_encoders_needed) {
    RTC_LOG(LS_INFO) << "[SEA] InitEncode: Single-encoder mode";
    int ret = encoder_context->encoder().InitEncode(&codec_, settings);
    if (ret >= 0) {
      stream_contexts_.emplace_back(
          /*parent=*/nullptr, std::move(encoder_context),
          /*framerate_controller=*/nullptr, /*stream_idx=*/0, codec_.width,
          codec_.height, /*is_paused=*/active_streams_count == 0);
      bypass_mode_ = true;

      DestroyStoredEncoders();
      inited_.store(1);
      return WEBRTC_VIDEO_CODEC_OK;
    }

    encoder_context->Release();
    encoder_context->encoder().RegisterEncodeCompleteCallback(
        encoded_complete_callback_);
    if (total_streams_count_ == 1) {
      RTC_LOG(LS_ERROR) << "[SEA] InitEncode: failed with error code: "
                        << WebRtcVideoCodecErrorToString(ret);
      return ret;
    }
    RTC_LOG(LS_WARNING) << "[SEA] InitEncode: failed with error code: "
                        << WebRtcVideoCodecErrorToString(ret)
                        << ". Falling back to multi-encoder mode.";
  }

  // Multi-encoder simulcast or singlecast (deactivated layers).
  std::vector<uint32_t> stream_start_bitrate_kbps =
      GetStreamStartBitratesKbps(env_, codec_);

  for (int stream_idx = 0; stream_idx < total_streams_count_; ++stream_idx) {
    if (!is_legacy_singlecast && !codec_.simulcastStream[stream_idx].active) {
      continue;
    }

    if (encoder_context == nullptr) {
      encoder_context = FetchOrCreateEncoderContext(
          /*is_lowest_quality_stream=*/stream_idx == lowest_quality_stream_idx);
    }
    if (encoder_context == nullptr) {
      Release();
      return WEBRTC_VIDEO_CODEC_MEMORY;
    }

    VideoCodec stream_codec = MakeStreamCodec(
        codec_, stream_idx, stream_start_bitrate_kbps[stream_idx],
        /*is_lowest_quality_stream=*/stream_idx == lowest_quality_stream_idx,
        /*is_highest_quality_stream=*/stream_idx == highest_quality_stream_idx);

    RTC_LOG(LS_INFO) << "[SEA] Multi-encoder mode: initializing stream: "
                     << stream_idx << ", active: "
                     << (codec_.simulcastStream[stream_idx].active ? "true"
                                                                   : "false");
    int ret = encoder_context->encoder().InitEncode(&stream_codec, settings);
    if (ret < 0) {
      encoder_context.reset();
      Release();
      RTC_LOG(LS_ERROR) << "[SEA] InitEncode: failed with error code: "
                        << WebRtcVideoCodecErrorToString(ret);
      return ret;
    }

    // Intercept frame encode complete callback only for upper streams, where
    // we need to set a correct stream index. Set `parent` to nullptr for the
    // lowest stream to bypass the callback.
    SimulcastEncoderAdapter* parent = stream_idx > 0 ? this : nullptr;

    bool is_paused = stream_start_bitrate_kbps[stream_idx] == 0;
    stream_contexts_.emplace_back(
        parent, std::move(encoder_context),
        std::make_unique<FramerateController>(stream_codec.maxFramerate),
        stream_idx, stream_codec.width, stream_codec.height, is_paused);
    encoder_context = nullptr;
  }

  // To save memory, don't store encoders that we don't use.
  DestroyStoredEncoders();

  inited_.store(1);
  return WEBRTC_VIDEO_CODEC_OK;
}

int SimulcastEncoderAdapter::Encode(
    const VideoFrame& input_image,
    const std::vector<VideoFrameType>* frame_types) {
  RTC_DCHECK_RUN_ON(&encoder_queue_);

  if (!Initialized()) {
    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  }
  if (encoded_complete_callback_ == nullptr) {
    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  }

  if (encoder_info_override_.requested_resolution_alignment()) {
    const int alignment =
        *encoder_info_override_.requested_resolution_alignment();
    if (input_image.width() % alignment != 0 ||
        input_image.height() % alignment != 0) {
      RTC_LOG(LS_WARNING) << "Frame " << input_image.width() << "x"
                          << input_image.height() << " not divisible by "
                          << alignment;
      return WEBRTC_VIDEO_CODEC_ERROR;
    }
    if (encoder_info_override_.apply_alignment_to_all_simulcast_layers()) {
      for (const auto& layer : stream_contexts_) {
        if (layer.width() % alignment != 0 || layer.height() % alignment != 0) {
          RTC_LOG(LS_WARNING)
              << "Codec " << layer.width() << "x" << layer.height()
              << " not divisible by " << alignment;
          return WEBRTC_VIDEO_CODEC_ERROR;
        }
      }
    }
  }

  bool is_keyframe_needed = false;
  for (const auto& layer : stream_contexts_) {
    if (layer.is_keyframe_needed()) {
      // This is legacy behavior, generating a keyframe on all layers
      // when generating one for a layer that became active for the first time
      // or after being disabled.
      is_keyframe_needed = true;
      break;
    }
  }

  // Temporary thay may hold the result of texture to i420 buffer conversion.
  rtc::scoped_refptr<VideoFrameBuffer> src_buffer;
  int src_width = input_image.width();
  int src_height = input_image.height();

  for (auto& layer : stream_contexts_) {
    // Don't encode frames in resolutions that we don't intend to send.
    if (layer.is_paused()) {
      continue;
    }

    // Convert timestamp from RTP 90kHz clock.
    const Timestamp frame_timestamp =
        Timestamp::Micros((1000 * input_image.rtp_timestamp()) / 90);

    // If adapter is passed through and only one sw encoder does simulcast,
    // frame types for all streams should be passed to the encoder unchanged.
    // Otherwise a single per-encoder frame type is passed.
    std::vector<VideoFrameType> stream_frame_types(
        bypass_mode_
            ? std::max<unsigned char>(codec_.numberOfSimulcastStreams, 1)
            : 1,
        VideoFrameType::kVideoFrameDelta);

    bool keyframe_requested = false;
    if (is_keyframe_needed) {
      std::fill(stream_frame_types.begin(), stream_frame_types.end(),
                VideoFrameType::kVideoFrameKey);
      keyframe_requested = true;
    } else if (frame_types) {
      if (bypass_mode_) {
        // In bypass mode, we effectively pass on frame_types.
        RTC_DCHECK_EQ(frame_types->size(), stream_frame_types.size());
        stream_frame_types = *frame_types;
        keyframe_requested =
            absl::c_any_of(*frame_types, [](const VideoFrameType frame_type) {
              return frame_type == VideoFrameType::kVideoFrameKey;
            });
      } else {
        size_t stream_idx = static_cast<size_t>(layer.stream_idx());
        if (frame_types->size() >= stream_idx &&
            (*frame_types)[stream_idx] == VideoFrameType::kVideoFrameKey) {
          stream_frame_types[0] = VideoFrameType::kVideoFrameKey;
          keyframe_requested = true;
        }
      }
    }
    if (keyframe_requested) {
      layer.OnKeyframe(frame_timestamp);
    } else if (layer.ShouldDropFrame(frame_timestamp)) {
      continue;
    }

    // If scaling isn't required, because the input resolution
    // matches the destination or the input image is empty (e.g.
    // a keyframe request for encoders with internal camera
    // sources) or the source image has a native handle, pass the image on
    // directly. Otherwise, we'll scale it to match what the encoder expects
    // (below).
    // For texture frames, the underlying encoder is expected to be able to
    // correctly sample/scale the source texture.
    // TODO(perkj): ensure that works going forward, and figure out how this
    // affects webrtc:5683.
    if ((layer.width() == src_width && layer.height() == src_height) ||
        (input_image.video_frame_buffer()->type() ==
             VideoFrameBuffer::Type::kNative &&
         layer.encoder().GetEncoderInfo().supports_native_handle)) {
      int ret = layer.encoder().Encode(input_image, &stream_frame_types);
      if (ret != WEBRTC_VIDEO_CODEC_OK) {
        return ret;
      }
    } else {
      if (src_buffer == nullptr) {
        src_buffer = input_image.video_frame_buffer();
      }
      rtc::scoped_refptr<VideoFrameBuffer> dst_buffer =
          src_buffer->Scale(layer.width(), layer.height());
      if (!dst_buffer) {
        RTC_LOG(LS_ERROR) << "Failed to scale video frame";
        return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
      }

      // UpdateRect is not propagated to lower simulcast layers currently.
      // TODO(ilnik): Consider scaling UpdateRect together with the buffer.
      VideoFrame frame(input_image);
      frame.set_video_frame_buffer(dst_buffer);
      frame.set_rotation(webrtc::kVideoRotation_0);
      frame.set_update_rect(
          VideoFrame::UpdateRect{0, 0, frame.width(), frame.height()});
      int ret = layer.encoder().Encode(frame, &stream_frame_types);
      if (ret != WEBRTC_VIDEO_CODEC_OK) {
        return ret;
      }
    }
  }

  return WEBRTC_VIDEO_CODEC_OK;
}

int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback(
    EncodedImageCallback* callback) {
  RTC_DCHECK_RUN_ON(&encoder_queue_);
  encoded_complete_callback_ = callback;
  if (!stream_contexts_.empty() && stream_contexts_.front().stream_idx() == 0) {
    // Bypass frame encode complete callback for the lowest layer since there is
    // no need to override frame's spatial index.
    stream_contexts_.front().encoder().RegisterEncodeCompleteCallback(callback);
  }
  return WEBRTC_VIDEO_CODEC_OK;
}

void SimulcastEncoderAdapter::SetRates(
    const RateControlParameters& parameters) {
  RTC_DCHECK_RUN_ON(&encoder_queue_);

  if (!Initialized()) {
    RTC_LOG(LS_WARNING) << "SetRates while not initialized";
    return;
  }

  if (parameters.framerate_fps < 1.0) {
    RTC_LOG(LS_WARNING) << "Invalid framerate: " << parameters.framerate_fps;
    return;
  }

  codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);

  if (bypass_mode_) {
    stream_contexts_.front().encoder().SetRates(parameters);
    return;
  }

  for (StreamContext& layer_context : stream_contexts_) {
    int stream_idx = layer_context.stream_idx();
    uint32_t stream_bitrate_kbps =
        parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;

    // Need a key frame if we have not sent this stream before.
    if (stream_bitrate_kbps > 0 && layer_context.is_paused()) {
      layer_context.set_is_keyframe_needed();
    }
    layer_context.set_is_paused(stream_bitrate_kbps == 0);

    // Slice the temporal layers out of the full allocation and pass it on to
    // the encoder handling the current simulcast stream.
    RateControlParameters stream_parameters = parameters;
    stream_parameters.bitrate = VideoBitrateAllocation();
    for (int i = 0; i < kMaxTemporalStreams; ++i) {
      if (parameters.bitrate.HasBitrate(stream_idx, i)) {
        stream_parameters.bitrate.SetBitrate(
            0, i, parameters.bitrate.GetBitrate(stream_idx, i));
      }
    }

    // Assign link allocation proportionally to spatial layer allocation.
    if (!parameters.bandwidth_allocation.IsZero() &&
        parameters.bitrate.get_sum_bps() > 0) {
      stream_parameters.bandwidth_allocation =
          DataRate::BitsPerSec((parameters.bandwidth_allocation.bps() *
                                stream_parameters.bitrate.get_sum_bps()) /
                               parameters.bitrate.get_sum_bps());
      // Make sure we don't allocate bandwidth lower than target bitrate.
      if (stream_parameters.bandwidth_allocation.bps() <
          stream_parameters.bitrate.get_sum_bps()) {
        stream_parameters.bandwidth_allocation =
            DataRate::BitsPerSec(stream_parameters.bitrate.get_sum_bps());
      }
    }

    stream_parameters.framerate_fps = std::min<double>(
        parameters.framerate_fps,
        layer_context.target_fps().value_or(parameters.framerate_fps));

    layer_context.encoder().SetRates(stream_parameters);
  }
}

void SimulcastEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) {
  for (auto& c : stream_contexts_) {
    c.encoder().OnPacketLossRateUpdate(packet_loss_rate);
  }
}

void SimulcastEncoderAdapter::OnRttUpdate(int64_t rtt_ms) {
  for (auto& c : stream_contexts_) {
    c.encoder().OnRttUpdate(rtt_ms);
  }
}

void SimulcastEncoderAdapter::OnLossNotification(
    const LossNotification& loss_notification) {
  for (auto& c : stream_contexts_) {
    c.encoder().OnLossNotification(loss_notification);
  }
}

// TODO(brandtr): Add task checker to this member function, when all encoder
// callbacks are coming in on the encoder queue.
EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage(
    size_t stream_idx,
    const EncodedImage& encodedImage,
    const CodecSpecificInfo* codecSpecificInfo) {
  EncodedImage stream_image(encodedImage);
  CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;

  stream_image.SetSimulcastIndex(stream_idx);

  return encoded_complete_callback_->OnEncodedImage(stream_image,
                                                    &stream_codec_specific);
}

void SimulcastEncoderAdapter::OnDroppedFrame(size_t /* stream_idx */) {
  // Not yet implemented.
}

bool SimulcastEncoderAdapter::Initialized() const {
  return inited_.load() == 1;
}

void SimulcastEncoderAdapter::DestroyStoredEncoders() {
  RTC_DCHECK_RUN_ON(&encoder_queue_);
  while (!cached_encoder_contexts_.empty()) {
    cached_encoder_contexts_.pop_back();
  }
}

std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
SimulcastEncoderAdapter::FetchOrCreateEncoderContext(
    bool is_lowest_quality_stream) const {
  RTC_DCHECK_RUN_ON(&encoder_queue_);
  bool prefer_temporal_support = fallback_encoder_factory_ != nullptr &&
                                 is_lowest_quality_stream &&
                                 prefer_temporal_support_on_base_layer_;

  // Toggling of `prefer_temporal_support` requires encoder recreation. Find
  // and reuse encoder with desired `prefer_temporal_support`. Otherwise, if
  // there is no such encoder in the cache, create a new instance.
  auto encoder_context_iter =
      std::find_if(cached_encoder_contexts_.begin(),
                   cached_encoder_contexts_.end(), [&](auto& encoder_context) {
                     return encoder_context->prefer_temporal_support() ==
                            prefer_temporal_support;
                   });

  std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context;
  if (encoder_context_iter != cached_encoder_contexts_.end()) {
    encoder_context = std::move(*encoder_context_iter);
    cached_encoder_contexts_.erase(encoder_context_iter);
  } else {
    std::unique_ptr<VideoEncoder> primary_encoder =
        primary_encoder_factory_->Create(env_, video_format_);

    std::unique_ptr<VideoEncoder> fallback_encoder;
    if (fallback_encoder_factory_ != nullptr) {
      fallback_encoder = fallback_encoder_factory_->Create(env_, video_format_);
    }

    std::unique_ptr<VideoEncoder> encoder;
    VideoEncoder::EncoderInfo primary_info;
    VideoEncoder::EncoderInfo fallback_info;

    if (primary_encoder != nullptr) {
      primary_info = primary_encoder->GetEncoderInfo();
      fallback_info = primary_info;

      if (fallback_encoder == nullptr) {
        encoder = std::move(primary_encoder);
      } else {
        encoder = CreateVideoEncoderSoftwareFallbackWrapper(
            env_, std::move(fallback_encoder), std::move(primary_encoder),
            prefer_temporal_support);
      }
    } else if (fallback_encoder != nullptr) {
      RTC_LOG(LS_WARNING) << "Failed to create primary " << video_format_.name
                          << " encoder. Use fallback encoder.";
      fallback_info = fallback_encoder->GetEncoderInfo();
      primary_info = fallback_info;
      encoder = std::move(fallback_encoder);
    } else {
      RTC_LOG(LS_ERROR) << "Failed to create primary and fallback "
                        << video_format_.name << " encoders.";
      return nullptr;
    }

    encoder_context = std::make_unique<SimulcastEncoderAdapter::EncoderContext>(
        std::move(encoder), prefer_temporal_support, primary_info,
        fallback_info);
  }

  encoder_context->encoder().RegisterEncodeCompleteCallback(
      encoded_complete_callback_);
  return encoder_context;
}

webrtc::VideoCodec SimulcastEncoderAdapter::MakeStreamCodec(
    const webrtc::VideoCodec& codec,
    int stream_idx,
    uint32_t start_bitrate_kbps,
    bool is_lowest_quality_stream,
    bool is_highest_quality_stream) {
  webrtc::VideoCodec codec_params = codec;
  const SimulcastStream& stream_params = codec.simulcastStream[stream_idx];

  codec_params.numberOfSimulcastStreams = 0;
  codec_params.width = stream_params.width;
  codec_params.height = stream_params.height;
  codec_params.maxBitrate = stream_params.maxBitrate;
  codec_params.minBitrate = stream_params.minBitrate;
  codec_params.maxFramerate = stream_params.maxFramerate;
  codec_params.qpMax = stream_params.qpMax;
  codec_params.active = stream_params.active;
  // By default, `scalability_mode` comes from SimulcastStream when
  // SimulcastEncoderAdapter is used. This allows multiple encodings of L1Tx,
  // but SimulcastStream currently does not support multiple spatial layers.
  std::optional<ScalabilityMode> scalability_mode =
      stream_params.GetScalabilityMode();
  // To support the full set of scalability modes in the event that this is the
  // only active encoding, prefer VideoCodec::GetScalabilityMode() if all other
  // encodings are inactive.
  bool only_active_stream = true;
  for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
    if (i != stream_idx && codec.simulcastStream[i].active) {
      only_active_stream = false;
      break;
    }
  }
  if (codec.GetScalabilityMode().has_value() && only_active_stream) {
    scalability_mode = codec.GetScalabilityMode();
  }
  if (scalability_mode.has_value()) {
    codec_params.SetScalabilityMode(*scalability_mode);
  }
  // Settings that are based on stream/resolution.
  if (is_lowest_quality_stream) {
    // Settings for lowest spatial resolutions.
    if (codec.mode == VideoCodecMode::kRealtimeVideo &&
        boost_base_layer_quality_) {
      codec_params.qpMax = kLowestResMaxQp;
    }
  }
  if (codec.codecType == webrtc::kVideoCodecVP8) {
    codec_params.VP8()->numberOfTemporalLayers =
        stream_params.numberOfTemporalLayers;
    if (!is_highest_quality_stream) {
      // For resolutions below CIF, set the codec `complexity` parameter to
      // kComplexityHigher, which maps to cpu_used = -4.
      int pixels_per_frame = codec_params.width * codec_params.height;
      if (pixels_per_frame < 352 * 288) {
        codec_params.SetVideoEncoderComplexity(
            webrtc::VideoCodecComplexity::kComplexityHigher);
      }
      // Turn off denoising for all streams but the highest resolution.
      codec_params.VP8()->denoisingOn = false;
    }
  } else if (codec.codecType == webrtc::kVideoCodecH264) {
    codec_params.H264()->numberOfTemporalLayers =
        stream_params.numberOfTemporalLayers;
  } else if (codec.codecType == webrtc::kVideoCodecVP9 &&
             scalability_mode.has_value() && !only_active_stream) {
    // If VP9 simulcast then explicitly set a single spatial layer for each
    // simulcast stream.
    codec_params.VP9()->numberOfSpatialLayers = 1;
    codec_params.VP9()->numberOfTemporalLayers =
        stream_params.GetNumberOfTemporalLayers();
    codec_params.VP9()->interLayerPred = InterLayerPredMode::kOff;
    codec_params.spatialLayers[0] = stream_params;
  }

  // Cap start bitrate to the min bitrate in order to avoid strange codec
  // behavior.
  codec_params.startBitrate =
      std::max(stream_params.minBitrate, start_bitrate_kbps);

  // Legacy screenshare mode is only enabled for the first simulcast layer
  codec_params.legacy_conference_mode =
      codec.legacy_conference_mode && stream_idx == 0;

  return codec_params;
}

void SimulcastEncoderAdapter::OverrideFromFieldTrial(
    VideoEncoder::EncoderInfo* info) const {
  if (encoder_info_override_.requested_resolution_alignment()) {
    info->requested_resolution_alignment =
        std::lcm(info->requested_resolution_alignment,
                 *encoder_info_override_.requested_resolution_alignment());
    info->apply_alignment_to_all_simulcast_layers =
        info->apply_alignment_to_all_simulcast_layers ||
        encoder_info_override_.apply_alignment_to_all_simulcast_layers();
  }
  // Override resolution bitrate limits unless they're set already.
  if (info->resolution_bitrate_limits.empty() &&
      !encoder_info_override_.resolution_bitrate_limits().empty()) {
    info->resolution_bitrate_limits =
        encoder_info_override_.resolution_bitrate_limits();
  }
}

VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
  if (stream_contexts_.size() == 1) {
    // Not using simulcast adapting functionality, just pass through.
    VideoEncoder::EncoderInfo info =
        stream_contexts_.front().encoder().GetEncoderInfo();
    OverrideFromFieldTrial(&info);
    return info;
  }

  VideoEncoder::EncoderInfo encoder_info;
  encoder_info.implementation_name = "SimulcastEncoderAdapter";
  encoder_info.requested_resolution_alignment = 1;
  encoder_info.apply_alignment_to_all_simulcast_layers = false;
  encoder_info.supports_native_handle = true;
  encoder_info.scaling_settings.thresholds = std::nullopt;

  if (stream_contexts_.empty()) {
    // GetEncoderInfo queried before InitEncode. Only alignment info is needed
    // to be filled.
    // Create one encoder and query it.

    std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context =
        FetchOrCreateEncoderContext(/*is_lowest_quality_stream=*/true);
    if (encoder_context == nullptr) {
      return encoder_info;
    }

    const VideoEncoder::EncoderInfo& primary_info =
        encoder_context->PrimaryInfo();
    const VideoEncoder::EncoderInfo& fallback_info =
        encoder_context->FallbackInfo();

    encoder_info.requested_resolution_alignment =
        std::lcm(primary_info.requested_resolution_alignment,
                 fallback_info.requested_resolution_alignment);

    encoder_info.apply_alignment_to_all_simulcast_layers =
        primary_info.apply_alignment_to_all_simulcast_layers ||
        fallback_info.apply_alignment_to_all_simulcast_layers;

    if (!primary_info.supports_simulcast || !fallback_info.supports_simulcast) {
      encoder_info.apply_alignment_to_all_simulcast_layers = true;
    }

    cached_encoder_contexts_.emplace_back(std::move(encoder_context));

    OverrideFromFieldTrial(&encoder_info);
    return encoder_info;
  }

  encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
  std::vector<std::string> encoder_names;

  for (size_t i = 0; i < stream_contexts_.size(); ++i) {
    VideoEncoder::EncoderInfo encoder_impl_info =
        stream_contexts_[i].encoder().GetEncoderInfo();

    // Encoder name indicates names of all active sub-encoders.
    if (!stream_contexts_[i].is_paused()) {
      encoder_names.push_back(encoder_impl_info.implementation_name);
    }
    if (i == 0) {
      encoder_info.supports_native_handle =
          encoder_impl_info.supports_native_handle;
      encoder_info.has_trusted_rate_controller =
          encoder_impl_info.has_trusted_rate_controller;
      encoder_info.is_hardware_accelerated =
          encoder_impl_info.is_hardware_accelerated;
      encoder_info.is_qp_trusted = encoder_impl_info.is_qp_trusted;
    } else {
      // Native handle supported if any encoder supports it.
      encoder_info.supports_native_handle |=
          encoder_impl_info.supports_native_handle;

      // Trusted rate controller only if all encoders have it.
      encoder_info.has_trusted_rate_controller &=
          encoder_impl_info.has_trusted_rate_controller;

      // Uses hardware support if any of the encoders uses it.
      // For example, if we are having issues with down-scaling due to
      // pipelining delay in HW encoders we need higher encoder usage
      // thresholds in CPU adaptation.
      encoder_info.is_hardware_accelerated |=
          encoder_impl_info.is_hardware_accelerated;

      // Treat QP from frame/slice/tile header as average QP only if all
      // encoders report it as average QP.
      encoder_info.is_qp_trusted =
          encoder_info.is_qp_trusted.value_or(true) &&
          encoder_impl_info.is_qp_trusted.value_or(true);
    }
    encoder_info.fps_allocation[i] = encoder_impl_info.fps_allocation[0];
    encoder_info.requested_resolution_alignment =
        std::lcm(encoder_info.requested_resolution_alignment,
                 encoder_impl_info.requested_resolution_alignment);
    // request alignment on all layers if any of the encoders may need it, or
    // if any non-top layer encoder requests a non-trivial alignment.
    if (encoder_impl_info.apply_alignment_to_all_simulcast_layers ||
        (encoder_impl_info.requested_resolution_alignment > 1 &&
         (codec_.simulcastStream[i].height < codec_.height ||
          codec_.simulcastStream[i].width < codec_.width))) {
      encoder_info.apply_alignment_to_all_simulcast_layers = true;
    }
  }

  if (!encoder_names.empty()) {
    rtc::StringBuilder implementation_name_builder(" (");
    implementation_name_builder << StrJoin(encoder_names, ", ");
    implementation_name_builder << ")";
    encoder_info.implementation_name += implementation_name_builder.Release();
  }

  OverrideFromFieldTrial(&encoder_info);

  return encoder_info;
}

}  // namespace webrtc