api/video/encoded_image.h

/*
 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef API_VIDEO_ENCODED_IMAGE_H_
#define API_VIDEO_ENCODED_IMAGE_H_

#include <stdint.h>

#include <cstddef>
#include <map>
#include <optional>
#include <utility>

#include "api/ref_count.h"
#include "api/rtp_packet_infos.h"
#include "api/scoped_refptr.h"
#include "api/units/timestamp.h"
#include "api/video/color_space.h"
#include "api/video/corruption_detection/corruption_detection_filter_settings.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_content_type.h"
#include "api/video/video_frame_type.h"
#include "api/video/video_rotation.h"
#include "api/video/video_timing.h"
#include "rtc_base/buffer.h"
#include "rtc_base/checks.h"
#include "rtc_base/system/rtc_export.h"

namespace webrtc {

// Abstract interface for buffer storage. Intended to support buffers owned by
// external encoders with special release requirements, e.g, java encoders with
// releaseOutputBuffer.
class EncodedImageBufferInterface : public RefCountInterface {
 public:
  using value_type = uint8_t;

  virtual const uint8_t* data() const = 0;
  virtual size_t size() const = 0;

  const uint8_t* begin() const { return data(); }
  const uint8_t* end() const { return data() + size(); }
};

// Basic implementation of EncodedImageBufferInterface.
class RTC_EXPORT EncodedImageBuffer : public EncodedImageBufferInterface {
 public:
  using iterator = Buffer::iterator;

  static scoped_refptr<EncodedImageBuffer> Create() { return Create(0); }
  static scoped_refptr<EncodedImageBuffer> Create(size_t size);
  static scoped_refptr<EncodedImageBuffer> Create(const uint8_t* data,
                                                  size_t size);
  static scoped_refptr<EncodedImageBuffer> Create(Buffer buffer);

  const uint8_t* data() const override;
  uint8_t* data();
  size_t size() const override;
  void Realloc(size_t t);

  iterator begin() { return buffer_.begin(); }
  iterator end() { return buffer_.end(); }

 protected:
  explicit EncodedImageBuffer(size_t size);
  EncodedImageBuffer(const uint8_t* data, size_t size);
  explicit EncodedImageBuffer(Buffer buffer);

  Buffer buffer_;
};

// TODO(bug.webrtc.org/9378): This is a legacy api class, which is slowly being
// cleaned up. Direct use of its members is strongly discouraged.
class RTC_EXPORT EncodedImage {
 public:
  // Peak signal to noise ratio, Y/U/V components.
  struct Psnr {
    double y = 0.0;
    double u = 0.0;
    double v = 0.0;
  };

  EncodedImage();
  EncodedImage(EncodedImage&&);
  EncodedImage(const EncodedImage&);

  ~EncodedImage();

  EncodedImage& operator=(EncodedImage&&);
  EncodedImage& operator=(const EncodedImage&);

  // Frame capture time in RTP timestamp representation (90kHz).
  void SetRtpTimestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; }
  uint32_t RtpTimestamp() const { return timestamp_rtp_; }

  void SetEncodeTime(int64_t encode_start_ms, int64_t encode_finish_ms);

  // Frame capture time in local time.
  Timestamp CaptureTime() const;

  // Frame capture time in ntp epoch time, i.e. time since 1st Jan 1900
  int64_t NtpTimeMs() const { return ntp_time_ms_; }

  // Every simulcast layer (= encoding) has its own encoder and RTP stream.
  // There can be no dependencies between different simulcast layers.
  std::optional<int> SimulcastIndex() const { return simulcast_index_; }
  void SetSimulcastIndex(std::optional<int> simulcast_index) {
    RTC_DCHECK_GE(simulcast_index.value_or(0), 0);
    RTC_DCHECK_LT(simulcast_index.value_or(0), kMaxSimulcastStreams);
    simulcast_index_ = simulcast_index;
  }

  const std::optional<Timestamp>& PresentationTimestamp() const {
    return presentation_timestamp_;
  }
  void SetPresentationTimestamp(
      const std::optional<Timestamp>& presentation_timestamp) {
    presentation_timestamp_ = presentation_timestamp;
  }

  // Encoded images can have dependencies between spatial and/or temporal
  // layers, depending on the scalability mode used by the encoder. See diagrams
  // at https://w3c.github.io/webrtc-svc/#dependencydiagrams*.
  std::optional<int> SpatialIndex() const { return spatial_index_; }
  void SetSpatialIndex(std::optional<int> spatial_index) {
    RTC_DCHECK_GE(spatial_index.value_or(0), 0);
    RTC_DCHECK_LT(spatial_index.value_or(0), kMaxSpatialLayers);
    spatial_index_ = spatial_index;
  }

  std::optional<int> TemporalIndex() const { return temporal_index_; }
  void SetTemporalIndex(std::optional<int> temporal_index) {
    RTC_DCHECK_GE(temporal_index.value_or(0), 0);
    RTC_DCHECK_LT(temporal_index.value_or(0), kMaxTemporalStreams);
    temporal_index_ = temporal_index;
  }

  // These methods can be used to set/get size of subframe with spatial index
  // `spatial_index` on encoded frames that consist of multiple spatial layers.
  std::optional<size_t> SpatialLayerFrameSize(int spatial_index) const;
  void SetSpatialLayerFrameSize(int spatial_index, size_t size_bytes);

  const webrtc::ColorSpace* ColorSpace() const {
    return color_space_ ? &*color_space_ : nullptr;
  }
  void SetColorSpace(const std::optional<webrtc::ColorSpace>& color_space) {
    color_space_ = color_space;
  }

  std::optional<VideoPlayoutDelay> PlayoutDelay() const {
    return playout_delay_;
  }

  void SetPlayoutDelay(std::optional<VideoPlayoutDelay> playout_delay) {
    playout_delay_ = playout_delay;
  }

  // These methods along with the private member video_frame_tracking_id_ are
  // meant for media quality testing purpose only.
  std::optional<uint16_t> VideoFrameTrackingId() const {
    return video_frame_tracking_id_;
  }
  void SetVideoFrameTrackingId(std::optional<uint16_t> tracking_id) {
    video_frame_tracking_id_ = tracking_id;
  }

  const RtpPacketInfos& PacketInfos() const { return packet_infos_; }
  void SetPacketInfos(RtpPacketInfos packet_infos) {
    packet_infos_ = std::move(packet_infos);
  }

  bool RetransmissionAllowed() const { return retransmission_allowed_; }
  void SetRetransmissionAllowed(bool retransmission_allowed) {
    retransmission_allowed_ = retransmission_allowed;
  }

  size_t size() const { return size_; }
  void set_size(size_t new_size) {
    // Allow set_size(0) even if we have no buffer.
    RTC_DCHECK_LE(new_size, new_size == 0 ? 0 : capacity());
    size_ = new_size;
  }

  void SetEncodedData(scoped_refptr<EncodedImageBufferInterface> encoded_data) {
    encoded_data_ = encoded_data;
    size_ = encoded_data->size();
  }

  void ClearEncodedData() {
    encoded_data_ = nullptr;
    size_ = 0;
  }

  scoped_refptr<EncodedImageBufferInterface> GetEncodedData() const {
    return encoded_data_;
  }

  const uint8_t* data() const {
    return encoded_data_ ? encoded_data_->data() : nullptr;
  }

  const uint8_t* begin() const { return data(); }
  const uint8_t* end() const { return data() + size(); }

  // Returns whether the encoded image can be considered to be of target
  // quality.
  [[deprecated]] bool IsAtTargetQuality() const { return at_target_quality_; }

  // Sets that the encoded image can be considered to be of target quality to
  // true or false.
  [[deprecated]] void SetAtTargetQuality(bool at_target_quality) {
    at_target_quality_ = at_target_quality;
  }

  // Returns whether the frame that was encoded is a steady-state refresh frame
  // intended to improve the visual quality.
  bool IsSteadyStateRefreshFrame() const {
    return is_steady_state_refresh_frame_;
  }

  void SetIsSteadyStateRefreshFrame(bool refresh_frame) {
    is_steady_state_refresh_frame_ = refresh_frame;
  }

  // TODO: webrtc:472264461 - Switch downstream projects to frame_type() and
  // remove this getter.
  VideoFrameType FrameType() const { return _frameType; }

  // TODO: webrtc:472264461 - Switch downstream projects to set_frame_type() and
  // remove this setter.
  void SetFrameType(VideoFrameType frame_type) { _frameType = frame_type; }

  VideoFrameType frame_type() const { return _frameType; }
  void set_frame_type(VideoFrameType frame_type) { _frameType = frame_type; }

  bool IsKey() const { return _frameType == VideoFrameType::kVideoFrameKey; }
  bool IsDelta() const {
    return _frameType == VideoFrameType::kVideoFrameDelta;
  }

  VideoContentType contentType() const { return content_type_; }
  VideoRotation rotation() const { return rotation_; }

  std::optional<CorruptionDetectionFilterSettings>
  corruption_detection_filter_settings() const {
    return corruption_detection_filter_settings_;
  }
  void set_corruption_detection_filter_settings(
      const CorruptionDetectionFilterSettings& settings) {
    corruption_detection_filter_settings_ = settings;
  }

  uint32_t _encodedWidth = 0;
  uint32_t _encodedHeight = 0;
  // NTP time of the capture time in local timebase in milliseconds.
  // TODO(minyue): make this member private.
  int64_t ntp_time_ms_ = 0;
  int64_t capture_time_ms_ = 0;
  VideoFrameType _frameType = VideoFrameType::kVideoFrameDelta;
  VideoRotation rotation_ = kVideoRotation_0;
  VideoContentType content_type_ = VideoContentType::UNSPECIFIED;
  int qp_ = -1;  // Quantizer value.

  struct Timing {
    uint8_t flags = VideoSendTiming::kInvalid;
    int64_t encode_start_ms = 0;
    int64_t encode_finish_ms = 0;
    int64_t packetization_finish_ms = 0;
    int64_t pacer_exit_ms = 0;
    int64_t network_timestamp_ms = 0;
    int64_t network2_timestamp_ms = 0;
    int64_t receive_start_ms = 0;
    int64_t receive_finish_ms = 0;
  } timing_;
  EncodedImage::Timing video_timing() const { return timing_; }
  EncodedImage::Timing* video_timing_mutable() { return &timing_; }

  std::optional<Psnr> psnr() const { return psnr_; }
  void set_psnr(std::optional<Psnr> psnr) { psnr_ = psnr; }

  void set_end_of_temporal_unit(bool is_end_of_temporal_unit) {
    is_end_of_temporal_unit_ = is_end_of_temporal_unit;
  }
  std::optional<bool> is_end_of_temporal_unit() const {
    return is_end_of_temporal_unit_;
  }

 private:
  size_t capacity() const { return encoded_data_ ? encoded_data_->size() : 0; }

  // When set, indicates that all future frames will be constrained with those
  // limits until the application indicates a change again.
  std::optional<VideoPlayoutDelay> playout_delay_;

  scoped_refptr<EncodedImageBufferInterface> encoded_data_;
  size_t size_ = 0;  // Size of encoded frame data.
  uint32_t timestamp_rtp_ = 0;
  std::optional<int> simulcast_index_;
  std::optional<Timestamp> presentation_timestamp_;
  std::optional<int> spatial_index_;
  std::optional<int> temporal_index_;
  std::map<int, size_t> spatial_layer_frame_size_bytes_;
  std::optional<webrtc::ColorSpace> color_space_;
  // This field is meant for media quality testing purpose only. When enabled it
  // carries the VideoFrame id field from the sender to the receiver.
  std::optional<uint16_t> video_frame_tracking_id_;
  // Information about packets used to assemble this video frame. This is needed
  // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's
  // MediaStreamTrack, in order to implement getContributingSources(). See:
  // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
  RtpPacketInfos packet_infos_;
  bool retransmission_allowed_ = true;
  // True if the encoded image can be considered to be of target quality.
  bool at_target_quality_ = false;
  // True if the frame that was encoded is a steady-state refresh frame intended
  // to improve the visual quality.
  bool is_steady_state_refresh_frame_ = false;

  // Filter settings for corruption detection suggested by the encoder
  // implementation, if any. Otherwise generic per-codec-type settings will be
  // used.
  std::optional<CorruptionDetectionFilterSettings>
      corruption_detection_filter_settings_;

  // Encoders may compute PSNR for a frame.
  std::optional<Psnr> psnr_;

  // If set, indicates whether this image is the last encoding of the temporal
  // unit identified by the RTP timestamp field, including any potentially
  // dropped layers.
  std::optional<bool> is_end_of_temporal_unit_;
};

}  // namespace webrtc

#endif  // API_VIDEO_ENCODED_IMAGE_H_