modules/rtp_rtcp/source/rtp_sender_video.cc

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/rtp_rtcp/source/rtp_sender_video.h"

#include <stdlib.h>
#include <string.h>

#include <algorithm>
#include <limits>
#include <memory>
#include <string>
#include <utility>

#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "api/transport/rtp/dependency_descriptor.h"
#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_format.h"
#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "modules/rtp_rtcp/source/time_util.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/trace_event.h"

namespace webrtc {

namespace {
constexpr size_t kRedForFecHeaderLength = 1;
constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;

// This is experimental field trial to exclude transport sequence number from
// FEC packets and should only be used in conjunction with datagram transport.
// Datagram transport removes transport sequence numbers from RTP packets and
// uses datagram feedback loop to re-generate RTCP feedback packets, but FEC
// contorol packets are calculated before sequence number is removed and as a
// result recovered packets will be corrupt unless we also remove transport
// sequence number during FEC calculation.
//
// TODO(sukhanov): We need to find find better way to implement FEC with
// datagram transport, probably moving FEC to datagram integration layter. We
// should also remove special field trial once we switch datagram path from
// RTCConfiguration flags to field trial and use the same field trial for FEC
// workaround.
const char kExcludeTransportSequenceNumberFromFecFieldTrial[] =
    "WebRTC-ExcludeTransportSequenceNumberFromFec";

void BuildRedPayload(const RtpPacketToSend& media_packet,
                     RtpPacketToSend* red_packet) {
  uint8_t* red_payload = red_packet->AllocatePayload(
      kRedForFecHeaderLength + media_packet.payload_size());
  RTC_DCHECK(red_payload);
  red_payload[0] = media_packet.PayloadType();

  auto media_payload = media_packet.payload();
  memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
         media_payload.size());
}

void AddRtpHeaderExtensions(
    const RTPVideoHeader& video_header,
    const absl::optional<PlayoutDelay>& playout_delay,
    const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
    FrameDependencyStructure* video_structure,
    bool set_video_rotation,
    bool set_color_space,
    bool set_frame_marking,
    bool first_packet,
    bool last_packet,
    RtpPacketToSend* packet) {
  // Color space requires two-byte header extensions if HDR metadata is
  // included. Therefore, it's best to add this extension first so that the
  // other extensions in the same packet are written as two-byte headers at
  // once.
  if (last_packet && set_color_space && video_header.color_space)
    packet->SetExtension<ColorSpaceExtension>(video_header.color_space.value());

  if (last_packet && set_video_rotation)
    packet->SetExtension<VideoOrientation>(video_header.rotation);

  // Report content type only for key frames.
  if (last_packet &&
      video_header.frame_type == VideoFrameType::kVideoFrameKey &&
      video_header.content_type != VideoContentType::UNSPECIFIED)
    packet->SetExtension<VideoContentTypeExtension>(video_header.content_type);

  if (last_packet &&
      video_header.video_timing.flags != VideoSendTiming::kInvalid)
    packet->SetExtension<VideoTimingExtension>(video_header.video_timing);

  // If transmitted, add to all packets; ack logic depends on this.
  if (playout_delay) {
    packet->SetExtension<PlayoutDelayLimits>(*playout_delay);
  }

  if (first_packet && absolute_capture_time) {
    packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time);
  }

  if (set_frame_marking) {
    FrameMarking frame_marking = video_header.frame_marking;
    frame_marking.start_of_frame = first_packet;
    frame_marking.end_of_frame = last_packet;
    packet->SetExtension<FrameMarkingExtension>(frame_marking);
  }

  if (video_header.generic) {
    bool extension_is_set = false;
    if (video_structure != nullptr) {
      DependencyDescriptor descriptor;
      descriptor.first_packet_in_frame = first_packet;
      descriptor.last_packet_in_frame = last_packet;
      descriptor.frame_number = video_header.generic->frame_id & 0xFFFF;
      descriptor.frame_dependencies.spatial_id =
          video_header.generic->spatial_index;
      descriptor.frame_dependencies.temporal_id =
          video_header.generic->temporal_index;
      for (int64_t dep : video_header.generic->dependencies) {
        descriptor.frame_dependencies.frame_diffs.push_back(
            video_header.generic->frame_id - dep);
      }
      descriptor.frame_dependencies.decode_target_indications =
          video_header.generic->decode_target_indications;
      RTC_DCHECK_EQ(
          descriptor.frame_dependencies.decode_target_indications.size(),
          video_structure->num_decode_targets);

      // To avoid extra structure copy, temporary share ownership of the
      // video_structure with the dependency descriptor.
      if (video_header.frame_type == VideoFrameType::kVideoFrameKey &&
          first_packet) {
        descriptor.attached_structure = absl::WrapUnique(video_structure);
      }
      extension_is_set = packet->SetExtension<RtpDependencyDescriptorExtension>(
          *video_structure, descriptor);

      // Remove the temporary shared ownership.
      descriptor.attached_structure.release();
    }

    // Do not use v0/v1 generic frame descriptor when v2 is stored.
    if (!extension_is_set) {
      RtpGenericFrameDescriptor generic_descriptor;
      generic_descriptor.SetFirstPacketInSubFrame(first_packet);
      generic_descriptor.SetLastPacketInSubFrame(last_packet);
      generic_descriptor.SetDiscardable(video_header.generic->discardable);

      if (first_packet) {
        generic_descriptor.SetFrameId(
            static_cast<uint16_t>(video_header.generic->frame_id));
        for (int64_t dep : video_header.generic->dependencies) {
          generic_descriptor.AddFrameDependencyDiff(
              video_header.generic->frame_id - dep);
        }

        uint8_t spatial_bimask = 1 << video_header.generic->spatial_index;
        generic_descriptor.SetSpatialLayersBitmask(spatial_bimask);

        generic_descriptor.SetTemporalLayer(
            video_header.generic->temporal_index);

        if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
          generic_descriptor.SetResolution(video_header.width,
                                           video_header.height);
        }
      }

      if (!packet->SetExtension<RtpGenericFrameDescriptorExtension01>(
              generic_descriptor)) {
        packet->SetExtension<RtpGenericFrameDescriptorExtension00>(
            generic_descriptor);
      }
    }
  }
}

bool MinimizeDescriptor(RTPVideoHeader* video_header) {
  if (auto* vp8 =
          absl::get_if<RTPVideoHeaderVP8>(&video_header->video_type_header)) {
    // Set minimum fields the RtpPacketizer is using to create vp8 packets.
    // nonReference is the only field that doesn't require extra space.
    bool non_reference = vp8->nonReference;
    vp8->InitRTPVideoHeaderVP8();
    vp8->nonReference = non_reference;
    return true;
  }
  // TODO(danilchap): Reduce vp9 codec specific descriptor too.
  return false;
}

bool IsBaseLayer(const RTPVideoHeader& video_header) {
  switch (video_header.codec) {
    case kVideoCodecVP8: {
      const auto& vp8 =
          absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
      return (vp8.temporalIdx == 0 || vp8.temporalIdx == kNoTemporalIdx);
    }
    case kVideoCodecVP9: {
      const auto& vp9 =
          absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
      return (vp9.temporal_idx == 0 || vp9.temporal_idx == kNoTemporalIdx);
    }
    case kVideoCodecH264:
      // TODO(kron): Implement logic for H264 once WebRTC supports temporal
      // layers for H264.
      break;
    default:
      break;
  }
  return true;
}

#if RTC_TRACE_EVENTS_ENABLED
const char* FrameTypeToString(VideoFrameType frame_type) {
  switch (frame_type) {
    case VideoFrameType::kEmptyFrame:
      return "empty";
    case VideoFrameType::kVideoFrameKey:
      return "video_key";
    case VideoFrameType::kVideoFrameDelta:
      return "video_delta";
    default:
      RTC_NOTREACHED();
      return "";
  }
}
#endif

bool IsNoopDelay(const PlayoutDelay& delay) {
  return delay.min_ms == -1 && delay.max_ms == -1;
}

}  // namespace

RTPSenderVideo::RTPSenderVideo(Clock* clock,
                               RTPSender* rtp_sender,
                               FlexfecSender* flexfec_sender,
                               PlayoutDelayOracle* playout_delay_oracle,
                               FrameEncryptorInterface* frame_encryptor,
                               bool require_frame_encryption,
                               bool enable_retransmit_all_layers,
                               const WebRtcKeyValueConfig& field_trials)
    : RTPSenderVideo([&] {
        Config config;
        config.clock = clock;
        config.rtp_sender = rtp_sender;
        config.flexfec_sender = flexfec_sender;
        config.frame_encryptor = frame_encryptor;
        config.require_frame_encryption = require_frame_encryption;
        config.enable_retransmit_all_layers = enable_retransmit_all_layers;
        config.field_trials = &field_trials;
        return config;
      }()) {}

RTPSenderVideo::RTPSenderVideo(const Config& config)
    : rtp_sender_(config.rtp_sender),
      clock_(config.clock),
      retransmission_settings_(
          config.enable_retransmit_all_layers
              ? kRetransmitAllLayers
              : (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)),
      last_rotation_(kVideoRotation_0),
      transmit_color_space_next_frame_(false),
      current_playout_delay_{-1, -1},
      playout_delay_pending_(false),
      red_payload_type_(config.red_payload_type),
      ulpfec_payload_type_(config.ulpfec_payload_type),
      flexfec_sender_(config.flexfec_sender),
      delta_fec_params_{0, 1, kFecMaskRandom},
      key_fec_params_{0, 1, kFecMaskRandom},
      fec_bitrate_(1000, RateStatistics::kBpsScale),
      video_bitrate_(1000, RateStatistics::kBpsScale),
      packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale),
      frame_encryptor_(config.frame_encryptor),
      require_frame_encryption_(config.require_frame_encryption),
      generic_descriptor_auth_experiment_(
          config.field_trials->Lookup("WebRTC-GenericDescriptorAuth")
              .find("Enabled") == 0),
      exclude_transport_sequence_number_from_fec_experiment_(
          config.field_trials
              ->Lookup(kExcludeTransportSequenceNumberFromFecFieldTrial)
              .find("Enabled") == 0),
      absolute_capture_time_sender_(config.clock) {}

RTPSenderVideo::~RTPSenderVideo() {}

void RTPSenderVideo::AppendAsRedMaybeWithUlpfec(
    std::unique_ptr<RtpPacketToSend> media_packet,
    bool protect_media_packet,
    std::vector<std::unique_ptr<RtpPacketToSend>>* packets) {
  std::unique_ptr<RtpPacketToSend> red_packet(
      new RtpPacketToSend(*media_packet));
  BuildRedPayload(*media_packet, red_packet.get());
  red_packet->SetPayloadType(*red_payload_type_);

  std::vector<std::unique_ptr<RedPacket>> fec_packets;
  if (ulpfec_enabled()) {
    if (protect_media_packet) {
      if (exclude_transport_sequence_number_from_fec_experiment_) {
        // See comments at the top of the file why experiment
        // "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in
        // conjunction with datagram transport.
        // TODO(sukhanov): We may also need to implement it for flexfec_sender
        // if we decide to keep this approach in the future.
        uint16_t transport_senquence_number;
        if (media_packet->GetExtension<webrtc::TransportSequenceNumber>(
                &transport_senquence_number)) {
          if (!media_packet->RemoveExtension(
                  webrtc::TransportSequenceNumber::kId)) {
            RTC_NOTREACHED()
                << "Failed to remove transport sequence number, packet="
                << media_packet->ToString();
          }
        }
      }

      ulpfec_generator_.AddRtpPacketAndGenerateFec(
          media_packet->Buffer(), media_packet->headers_size());
    }
    uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
    if (num_fec_packets > 0) {
      uint16_t first_fec_sequence_number =
          rtp_sender_->AllocateSequenceNumber(num_fec_packets);
      fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed(
          *red_payload_type_, *ulpfec_payload_type_, first_fec_sequence_number);
      RTC_DCHECK_EQ(num_fec_packets, fec_packets.size());
    }
  }

  // Send |red_packet| instead of |packet| for allocated sequence number.
  red_packet->set_packet_type(RtpPacketMediaType::kVideo);
  red_packet->set_allow_retransmission(media_packet->allow_retransmission());
  packets->emplace_back(std::move(red_packet));

  for (const auto& fec_packet : fec_packets) {
    // TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid
    // reparsing them.
    std::unique_ptr<RtpPacketToSend> rtp_packet(
        new RtpPacketToSend(*media_packet));
    RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length()));
    rtp_packet->set_capture_time_ms(media_packet->capture_time_ms());
    rtp_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection);
    rtp_packet->set_allow_retransmission(false);
    RTC_DCHECK_EQ(fec_packet->length(), rtp_packet->size());
    packets->emplace_back(std::move(rtp_packet));
  }
}

void RTPSenderVideo::GenerateAndAppendFlexfec(
    std::vector<std::unique_ptr<RtpPacketToSend>>* packets) {
  RTC_DCHECK(flexfec_sender_);

  if (flexfec_sender_->FecAvailable()) {
    std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
        flexfec_sender_->GetFecPackets();
    for (auto& fec_packet : fec_packets) {
      fec_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection);
      fec_packet->set_allow_retransmission(false);
      packets->emplace_back(std::move(fec_packet));
    }
  }
}

void RTPSenderVideo::LogAndSendToNetwork(
    std::vector<std::unique_ptr<RtpPacketToSend>> packets,
    size_t unpacketized_payload_size) {
  int64_t now_ms = clock_->TimeInMilliseconds();
#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
  for (const auto& packet : packets) {
    if (packet->packet_type() == RtpPacketMediaType::kForwardErrorCorrection) {
      const uint32_t ssrc = packet->Ssrc();
      BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoFecBitrate_kbps", now_ms,
                                      FecOverheadRate() / 1000, ssrc);
    }
  }
#endif

  {
    rtc::CritScope cs(&stats_crit_);
    size_t packetized_payload_size = 0;
    for (const auto& packet : packets) {
      switch (*packet->packet_type()) {
        case RtpPacketMediaType::kVideo:
          video_bitrate_.Update(packet->size(), now_ms);
          packetized_payload_size += packet->payload_size();
          break;
        case RtpPacketMediaType::kForwardErrorCorrection:
          fec_bitrate_.Update(packet->size(), clock_->TimeInMilliseconds());
          break;
        default:
          continue;
      }
    }
    // AV1 packetizer may produce less packetized bytes than unpacketized.
    if (packetized_payload_size >= unpacketized_payload_size) {
      packetization_overhead_bitrate_.Update(
          packetized_payload_size - unpacketized_payload_size,
          clock_->TimeInMilliseconds());
    }
  }

  rtp_sender_->EnqueuePackets(std::move(packets));
}

size_t RTPSenderVideo::FecPacketOverhead() const {
  if (flexfec_enabled())
    return flexfec_sender_->MaxPacketOverhead();

  size_t overhead = 0;
  if (red_enabled()) {
    // The RED overhead is due to a small header.
    overhead += kRedForFecHeaderLength;
  }
  if (ulpfec_enabled()) {
    // For ULPFEC, the overhead is the FEC headers plus RED for FEC header
    // (see above) plus anything in RTP header beyond the 12 bytes base header
    // (CSRC list, extensions...)
    // This reason for the header extensions to be included here is that
    // from an FEC viewpoint, they are part of the payload to be protected.
    // (The base RTP header is already protected by the FEC header.)
    overhead += ulpfec_generator_.MaxPacketOverhead() +
                (rtp_sender_->RtpHeaderLength() - kRtpHeaderSize);
  }
  return overhead;
}

void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params,
                                      const FecProtectionParams& key_params) {
  rtc::CritScope cs(&crit_);
  delta_fec_params_ = delta_params;
  key_fec_params_ = key_params;
}

absl::optional<uint32_t> RTPSenderVideo::FlexfecSsrc() const {
  if (flexfec_sender_) {
    return flexfec_sender_->ssrc();
  }
  return absl::nullopt;
}

void RTPSenderVideo::SetVideoStructure(
    const FrameDependencyStructure* video_structure) {
  RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
  if (video_structure == nullptr) {
    video_structure_ = nullptr;
    return;
  }
  // Simple sanity checks video structure is set up.
  RTC_DCHECK_GT(video_structure->num_decode_targets, 0);
  RTC_DCHECK_GT(video_structure->templates.size(), 0);

  int structure_id = 0;
  if (video_structure_) {
    if (*video_structure_ == *video_structure) {
      // Same structure (just a new key frame), no update required.
      return;
    }
    // When setting different video structure make sure structure_id is updated
    // so that templates from different structures do not collide.
    static constexpr int kMaxTemplates = 64;
    structure_id =
        (video_structure_->structure_id + video_structure_->templates.size()) %
        kMaxTemplates;
  }

  video_structure_ =
      std::make_unique<FrameDependencyStructure>(*video_structure);
  video_structure_->structure_id = structure_id;
  // TODO(bugs.webrtc.org/10342): Support chains.
  video_structure_->num_chains = 0;
}

bool RTPSenderVideo::SendVideo(
    int payload_type,
    absl::optional<VideoCodecType> codec_type,
    uint32_t rtp_timestamp,
    int64_t capture_time_ms,
    rtc::ArrayView<const uint8_t> payload,
    const RTPFragmentationHeader* fragmentation,
    RTPVideoHeader video_header,
    absl::optional<int64_t> expected_retransmission_time_ms) {
  #if RTC_TRACE_EVENTS_ENABLED
  TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type",
                          FrameTypeToString(video_header.frame_type));
  #endif
  RTC_CHECK_RUNS_SERIALIZED(&send_checker_);

  if (video_header.frame_type == VideoFrameType::kEmptyFrame)
    return true;

  if (payload.empty())
    return false;

  int32_t retransmission_settings = retransmission_settings_;
  if (codec_type == VideoCodecType::kVideoCodecH264) {
    // Backward compatibility for older receivers without temporal layer logic.
    retransmission_settings = kRetransmitBaseLayer | kRetransmitHigherLayers;
  }

  bool set_frame_marking =
      video_header.codec == kVideoCodecH264 &&
      video_header.frame_marking.temporal_id != kNoTemporalIdx;

  MaybeUpdateCurrentPlayoutDelay(video_header);
  if (video_header.frame_type == VideoFrameType::kVideoFrameKey &&
      !IsNoopDelay(current_playout_delay_)) {
    // Force playout delay on key-frames, if set.
    playout_delay_pending_ = true;
  }
  const absl::optional<PlayoutDelay> playout_delay =
      playout_delay_pending_
          ? absl::optional<PlayoutDelay>(current_playout_delay_)
          : absl::nullopt;

  // According to
  // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
  // ts_126114v120700p.pdf Section 7.4.5:
  // The MTSI client shall add the payload bytes as defined in this clause
  // onto the last RTP packet in each group of packets which make up a key
  // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
  // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
  // packet in each group of packets which make up another type of frame
  // (e.g. a P-Frame) only if the current value is different from the previous
  // value sent.
  // Set rotation when key frame or when changed (to follow standard).
  // Or when different from 0 (to follow current receiver implementation).
  bool set_video_rotation =
      video_header.frame_type == VideoFrameType::kVideoFrameKey ||
      video_header.rotation != last_rotation_ ||
      video_header.rotation != kVideoRotation_0;
  last_rotation_ = video_header.rotation;

  // Send color space when changed or if the frame is a key frame. Keep
  // sending color space information until the first base layer frame to
  // guarantee that the information is retrieved by the receiver.
  bool set_color_space;
  if (video_header.color_space != last_color_space_) {
    last_color_space_ = video_header.color_space;
    set_color_space = true;
    transmit_color_space_next_frame_ = !IsBaseLayer(video_header);
  } else {
    set_color_space =
        video_header.frame_type == VideoFrameType::kVideoFrameKey ||
        transmit_color_space_next_frame_;
    transmit_color_space_next_frame_ =
        transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false;
  }

  if (flexfec_enabled() || ulpfec_enabled()) {
    rtc::CritScope cs(&crit_);
    // FEC settings.
    const FecProtectionParams& fec_params =
        video_header.frame_type == VideoFrameType::kVideoFrameKey
            ? key_fec_params_
            : delta_fec_params_;
    if (flexfec_enabled())
      flexfec_sender_->SetFecParameters(fec_params);
    if (ulpfec_enabled())
      ulpfec_generator_.SetFecParameters(fec_params);
  }

  // Maximum size of packet including rtp headers.
  // Extra space left in case packet will be resent using fec or rtx.
  int packet_capacity = rtp_sender_->MaxRtpPacketSize() - FecPacketOverhead() -
                        (rtp_sender_->RtxStatus() ? kRtxHeaderSize : 0);

  std::unique_ptr<RtpPacketToSend> single_packet =
      rtp_sender_->AllocatePacket();
  RTC_DCHECK_LE(packet_capacity, single_packet->capacity());
  single_packet->SetPayloadType(payload_type);
  single_packet->SetTimestamp(rtp_timestamp);
  single_packet->set_capture_time_ms(capture_time_ms);

  const absl::optional<AbsoluteCaptureTime> absolute_capture_time =
      absolute_capture_time_sender_.OnSendPacket(
          AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
                                               single_packet->Csrcs()),
          single_packet->Timestamp(), kVideoPayloadTypeFrequency,
          Int64MsToUQ32x32(single_packet->capture_time_ms() + NtpOffsetMs()),
          /*estimated_capture_clock_offset=*/absl::nullopt);

  auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
  auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
  auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
  // Simplest way to estimate how much extensions would occupy is to set them.
  AddRtpHeaderExtensions(video_header, playout_delay, absolute_capture_time,
                         video_structure_.get(), set_video_rotation,
                         set_color_space, set_frame_marking,
                         /*first=*/true, /*last=*/true, single_packet.get());
  AddRtpHeaderExtensions(video_header, playout_delay, absolute_capture_time,
                         video_structure_.get(), set_video_rotation,
                         set_color_space, set_frame_marking,
                         /*first=*/true, /*last=*/false, first_packet.get());
  AddRtpHeaderExtensions(video_header, playout_delay, absolute_capture_time,
                         video_structure_.get(), set_video_rotation,
                         set_color_space, set_frame_marking,
                         /*first=*/false, /*last=*/false, middle_packet.get());
  AddRtpHeaderExtensions(video_header, playout_delay, absolute_capture_time,
                         video_structure_.get(), set_video_rotation,
                         set_color_space, set_frame_marking,
                         /*first=*/false, /*last=*/true, last_packet.get());

  RTC_DCHECK_GT(packet_capacity, single_packet->headers_size());
  RTC_DCHECK_GT(packet_capacity, first_packet->headers_size());
  RTC_DCHECK_GT(packet_capacity, middle_packet->headers_size());
  RTC_DCHECK_GT(packet_capacity, last_packet->headers_size());
  RtpPacketizer::PayloadSizeLimits limits;
  limits.max_payload_len = packet_capacity - middle_packet->headers_size();

  RTC_DCHECK_GE(single_packet->headers_size(), middle_packet->headers_size());
  limits.single_packet_reduction_len =
      single_packet->headers_size() - middle_packet->headers_size();

  RTC_DCHECK_GE(first_packet->headers_size(), middle_packet->headers_size());
  limits.first_packet_reduction_len =
      first_packet->headers_size() - middle_packet->headers_size();

  RTC_DCHECK_GE(last_packet->headers_size(), middle_packet->headers_size());
  limits.last_packet_reduction_len =
      last_packet->headers_size() - middle_packet->headers_size();

  rtc::ArrayView<const uint8_t> generic_descriptor_raw_00 =
      first_packet->GetRawExtension<RtpGenericFrameDescriptorExtension00>();
  rtc::ArrayView<const uint8_t> generic_descriptor_raw_01 =
      first_packet->GetRawExtension<RtpGenericFrameDescriptorExtension01>();

  if (!generic_descriptor_raw_00.empty() &&
      !generic_descriptor_raw_01.empty()) {
    RTC_LOG(LS_WARNING) << "Two versions of GFD extension used.";
    return false;
  }

  // Minimiazation of the vp8 descriptor may erase temporal_id, so save it.
  const uint8_t temporal_id = GetTemporalId(video_header);
  rtc::ArrayView<const uint8_t> generic_descriptor_raw =
      !generic_descriptor_raw_01.empty() ? generic_descriptor_raw_01
                                         : generic_descriptor_raw_00;
  if (!generic_descriptor_raw.empty()) {
    MinimizeDescriptor(&video_header);
  }

  if (video_header.frame_type == VideoFrameType::kVideoFrameKey ||
      (IsBaseLayer(video_header) &&
       !(video_header.generic.has_value() ? video_header.generic->discardable
                                          : false))) {
    // This frame has guaranteed delivery, no need to populate playout
    // delay extensions until it changes again.
    playout_delay_pending_ = false;
  }

  // TODO(benwright@webrtc.org) - Allocate enough to always encrypt inline.
  rtc::Buffer encrypted_video_payload;
  if (frame_encryptor_ != nullptr) {
    if (generic_descriptor_raw.empty()) {
      return false;
    }

    const size_t max_ciphertext_size =
        frame_encryptor_->GetMaxCiphertextByteSize(cricket::MEDIA_TYPE_VIDEO,
                                                   payload.size());
    encrypted_video_payload.SetSize(max_ciphertext_size);

    size_t bytes_written = 0;

    // Only enable header authentication if the field trial is enabled.
    rtc::ArrayView<const uint8_t> additional_data;
    if (generic_descriptor_auth_experiment_) {
      additional_data = generic_descriptor_raw;
    }

    if (frame_encryptor_->Encrypt(
            cricket::MEDIA_TYPE_VIDEO, first_packet->Ssrc(), additional_data,
            payload, encrypted_video_payload, &bytes_written) != 0) {
      return false;
    }

    encrypted_video_payload.SetSize(bytes_written);
    payload = encrypted_video_payload;
  } else if (require_frame_encryption_) {
    RTC_LOG(LS_WARNING)
        << "No FrameEncryptor is attached to this video sending stream but "
           "one is required since require_frame_encryptor is set";
  }

  std::unique_ptr<RtpPacketizer> packetizer = RtpPacketizer::Create(
      codec_type, payload, limits, video_header, fragmentation);

  // TODO(bugs.webrtc.org/10714): retransmission_settings_ should generally be
  // replaced by expected_retransmission_time_ms.has_value(). For now, though,
  // only VP8 with an injected frame buffer controller actually controls it.
  const bool allow_retransmission =
      expected_retransmission_time_ms.has_value()
          ? AllowRetransmission(temporal_id, retransmission_settings,
                                expected_retransmission_time_ms.value())
          : false;
  const size_t num_packets = packetizer->NumPackets();

  size_t unpacketized_payload_size;
  if (fragmentation && fragmentation->fragmentationVectorSize > 0) {
    unpacketized_payload_size = 0;
    for (uint16_t i = 0; i < fragmentation->fragmentationVectorSize; ++i) {
      unpacketized_payload_size += fragmentation->fragmentationLength[i];
    }
  } else {
    unpacketized_payload_size = payload.size();
  }

  if (num_packets == 0)
    return false;

  bool first_frame = first_frame_sent_();
  std::vector<std::unique_ptr<RtpPacketToSend>> rtp_packets;
  for (size_t i = 0; i < num_packets; ++i) {
    std::unique_ptr<RtpPacketToSend> packet;
    int expected_payload_capacity;
    // Choose right packet template:
    if (num_packets == 1) {
      packet = std::move(single_packet);
      expected_payload_capacity =
          limits.max_payload_len - limits.single_packet_reduction_len;
    } else if (i == 0) {
      packet = std::move(first_packet);
      expected_payload_capacity =
          limits.max_payload_len - limits.first_packet_reduction_len;
    } else if (i == num_packets - 1) {
      packet = std::move(last_packet);
      expected_payload_capacity =
          limits.max_payload_len - limits.last_packet_reduction_len;
    } else {
      packet = std::make_unique<RtpPacketToSend>(*middle_packet);
      expected_payload_capacity = limits.max_payload_len;
    }

    packet->set_first_packet_of_frame(i == 0);

    if (!packetizer->NextPacket(packet.get()))
      return false;
    RTC_DCHECK_LE(packet->payload_size(), expected_payload_capacity);
    if (!rtp_sender_->AssignSequenceNumber(packet.get()))
      return false;

    // No FEC protection for upper temporal layers, if used.
    bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx;

    packet->set_allow_retransmission(allow_retransmission);

    // Put packetization finish timestamp into extension.
    if (packet->HasExtension<VideoTimingExtension>()) {
      packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
    }

    if (red_enabled()) {
      AppendAsRedMaybeWithUlpfec(std::move(packet), protect_packet,
                                 &rtp_packets);
    } else {
      packet->set_packet_type(RtpPacketMediaType::kVideo);
      const RtpPacketToSend& media_packet = *packet;
      rtp_packets.emplace_back(std::move(packet));
      if (flexfec_enabled()) {
        // TODO(brandtr): Remove the FlexFEC code path when FlexfecSender
        // is wired up to PacedSender instead.
        if (protect_packet) {
          flexfec_sender_->AddRtpPacketAndGenerateFec(media_packet);
        }
        GenerateAndAppendFlexfec(&rtp_packets);
      }
    }

    if (first_frame) {
      if (i == 0) {
        RTC_LOG(LS_INFO)
            << "Sent first RTP packet of the first video frame (pre-pacer)";
      }
      if (i == num_packets - 1) {
        RTC_LOG(LS_INFO)
            << "Sent last RTP packet of the first video frame (pre-pacer)";
      }
    }
  }

  LogAndSendToNetwork(std::move(rtp_packets), unpacketized_payload_size);

  TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
                         rtp_timestamp);
  return true;
}

uint32_t RTPSenderVideo::VideoBitrateSent() const {
  rtc::CritScope cs(&stats_crit_);
  return video_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
}

uint32_t RTPSenderVideo::FecOverheadRate() const {
  rtc::CritScope cs(&stats_crit_);
  return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
}

uint32_t RTPSenderVideo::PacketizationOverheadBps() const {
  rtc::CritScope cs(&stats_crit_);
  return packetization_overhead_bitrate_.Rate(clock_->TimeInMilliseconds())
      .value_or(0);
}

bool RTPSenderVideo::AllowRetransmission(
    uint8_t temporal_id,
    int32_t retransmission_settings,
    int64_t expected_retransmission_time_ms) {
  if (retransmission_settings == kRetransmitOff)
    return false;

  rtc::CritScope cs(&stats_crit_);
  // Media packet storage.
  if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
      UpdateConditionalRetransmit(temporal_id,
                                  expected_retransmission_time_ms)) {
    retransmission_settings |= kRetransmitHigherLayers;
  }

  if (temporal_id == kNoTemporalIdx)
    return true;

  if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
    return true;

  if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
    return true;

  return false;
}

uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) {
  struct TemporalIdGetter {
    uint8_t operator()(const RTPVideoHeaderVP8& vp8) { return vp8.temporalIdx; }
    uint8_t operator()(const RTPVideoHeaderVP9& vp9) {
      return vp9.temporal_idx;
    }
    uint8_t operator()(const RTPVideoHeaderH264&) { return kNoTemporalIdx; }
    uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) {
      return kNoTemporalIdx;
    }
    uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; }
  };
  switch (header.codec) {
    case kVideoCodecH264:
      return header.frame_marking.temporal_id;
    default:
      return absl::visit(TemporalIdGetter(), header.video_type_header);
  }
}

bool RTPSenderVideo::UpdateConditionalRetransmit(
    uint8_t temporal_id,
    int64_t expected_retransmission_time_ms) {
  int64_t now_ms = clock_->TimeInMilliseconds();
  // Update stats for any temporal layer.
  TemporalLayerStats* current_layer_stats =
      &frame_stats_by_temporal_layer_[temporal_id];
  current_layer_stats->frame_rate_fp1000s.Update(1, now_ms);
  int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_ms;
  current_layer_stats->last_frame_time_ms = now_ms;

  // Conditional retransmit only applies to upper layers.
  if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
    if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
      // Too long since a retransmittable frame in this layer, enable NACK
      // protection.
      return true;
    } else {
      // Estimate when the next frame of any lower layer will be sent.
      const int64_t kUndefined = std::numeric_limits<int64_t>::max();
      int64_t expected_next_frame_time = kUndefined;
      for (int i = temporal_id - 1; i >= 0; --i) {
        TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
        absl::optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms);
        if (rate) {
          int64_t tl_next = stats->last_frame_time_ms + 1000000 / *rate;
          if (tl_next - now_ms > -expected_retransmission_time_ms &&
              tl_next < expected_next_frame_time) {
            expected_next_frame_time = tl_next;
          }
        }
      }

      if (expected_next_frame_time == kUndefined ||
          expected_next_frame_time - now_ms > expected_retransmission_time_ms) {
        // The next frame in a lower layer is expected at a later time (or
        // unable to tell due to lack of data) than a retransmission is
        // estimated to be able to arrive, so allow this packet to be nacked.
        return true;
      }
    }
  }

  return false;
}

void RTPSenderVideo::MaybeUpdateCurrentPlayoutDelay(
    const RTPVideoHeader& header) {
  if (IsNoopDelay(header.playout_delay)) {
    return;
  }

  PlayoutDelay requested_delay = header.playout_delay;

  if (requested_delay.min_ms > PlayoutDelayLimits::kMaxMs ||
      requested_delay.max_ms > PlayoutDelayLimits::kMaxMs) {
    RTC_DLOG(LS_ERROR)
        << "Requested playout delay values out of range, ignored";
    return;
  }
  if (requested_delay.max_ms != -1 &&
      requested_delay.min_ms > requested_delay.max_ms) {
    RTC_DLOG(LS_ERROR) << "Requested playout delay values out of order";
    return;
  }

  if (!playout_delay_pending_) {
    current_playout_delay_ = requested_delay;
    playout_delay_pending_ = true;
    return;
  }

  if ((requested_delay.min_ms == -1 ||
       requested_delay.min_ms == current_playout_delay_.min_ms) &&
      (requested_delay.max_ms == -1 ||
       requested_delay.max_ms == current_playout_delay_.max_ms)) {
    // No change, ignore.
    return;
  }

  if (requested_delay.min_ms == -1) {
    RTC_DCHECK_GE(requested_delay.max_ms, 0);
    requested_delay.min_ms =
        std::min(current_playout_delay_.min_ms, requested_delay.max_ms);
  }
  if (requested_delay.max_ms == -1) {
    requested_delay.max_ms =
        std::max(current_playout_delay_.max_ms, requested_delay.min_ms);
  }

  current_playout_delay_ = requested_delay;
  playout_delay_pending_ = true;
}

}  // namespace webrtc