media/base/media_channel.h

/*
 *  Copyright (c) 2004 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef MEDIA_BASE_MEDIA_CHANNEL_H_
#define MEDIA_BASE_MEDIA_CHANNEL_H_

#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>

#include "absl/types/optional.h"
#include "api/audio/audio_processing_statistics.h"
#include "api/audio_codecs/audio_encoder.h"
#include "api/audio_options.h"
#include "api/call/audio_sink.h"
#include "api/crypto/frame_decryptor_interface.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "api/frame_transformer_interface.h"
#include "api/media_stream_interface.h"
#include "api/rtc_error.h"
#include "api/rtp_parameters.h"
#include "api/rtp_sender_interface.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/transport/data_channel_transport_interface.h"
#include "api/transport/rtp/rtp_source.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "api/video/video_content_type.h"
#include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h"
#include "api/video/video_timing.h"
#include "api/video_codecs/scalability_mode.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "call/video_receive_stream.h"
#include "common_video/include/quality_limitation_reason.h"
#include "media/base/codec.h"
#include "media/base/media_constants.h"
#include "media/base/stream_params.h"
#include "modules/rtp_rtcp/include/report_block_data.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "rtc_base/async_packet_socket.h"
#include "rtc_base/buffer.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/dscp.h"
#include "rtc_base/logging.h"
#include "rtc_base/network_route.h"
#include "rtc_base/socket.h"
#include "rtc_base/string_encode.h"
#include "rtc_base/strings/string_builder.h"
#include "video/config/video_encoder_config.h"

namespace rtc {
class Timing;
}

namespace webrtc {
class VideoFrame;

webrtc::RTCError InvokeSetParametersCallback(SetParametersCallback& callback,
                                             RTCError error);

}  // namespace webrtc

namespace cricket {

class AudioSource;
class VideoCapturer;
struct RtpHeader;
struct VideoFormat;
class VideoMediaSendChannelInterface;
class VideoMediaReceiveChannelInterface;
class VoiceMediaSendChannelInterface;
class VoiceMediaReceiveChannelInterface;

const int kScreencastDefaultFps = 5;

template <class T>
static std::string ToStringIfSet(const char* key,
                                 const absl::optional<T>& val) {
  std::string str;
  if (val) {
    str = key;
    str += ": ";
    str += val ? rtc::ToString(*val) : "";
    str += ", ";
  }
  return str;
}

template <class T>
static std::string VectorToString(const std::vector<T>& vals) {
  rtc::StringBuilder ost;
  ost << "[";
  for (size_t i = 0; i < vals.size(); ++i) {
    if (i > 0) {
      ost << ", ";
    }
    ost << vals[i].ToString();
  }
  ost << "]";
  return ost.Release();
}

// Options that can be applied to a VideoMediaChannel or a VideoMediaEngine.
// Used to be flags, but that makes it hard to selectively apply options.
// We are moving all of the setting of options to structs like this,
// but some things currently still use flags.
struct VideoOptions {
  VideoOptions();
  ~VideoOptions();

  void SetAll(const VideoOptions& change) {
    SetFrom(&video_noise_reduction, change.video_noise_reduction);
    SetFrom(&screencast_min_bitrate_kbps, change.screencast_min_bitrate_kbps);
    SetFrom(&is_screencast, change.is_screencast);
  }

  bool operator==(const VideoOptions& o) const {
    return video_noise_reduction == o.video_noise_reduction &&
           screencast_min_bitrate_kbps == o.screencast_min_bitrate_kbps &&
           is_screencast == o.is_screencast;
  }
  bool operator!=(const VideoOptions& o) const { return !(*this == o); }

  std::string ToString() const {
    rtc::StringBuilder ost;
    ost << "VideoOptions {";
    ost << ToStringIfSet("noise reduction", video_noise_reduction);
    ost << ToStringIfSet("screencast min bitrate kbps",
                         screencast_min_bitrate_kbps);
    ost << ToStringIfSet("is_screencast ", is_screencast);
    ost << "}";
    return ost.Release();
  }

  // Enable denoising? This flag comes from the getUserMedia
  // constraint 'googNoiseReduction', and WebRtcVideoEngine passes it
  // on to the codec options. Disabled by default.
  absl::optional<bool> video_noise_reduction;
  // Force screencast to use a minimum bitrate. This flag comes from
  // the PeerConnection constraint 'googScreencastMinBitrate'. It is
  // copied to the encoder config by WebRtcVideoChannel.
  // TODO(https://crbug.com/1315155): Remove the ability to set it in Chromium
  // and delete this flag (it should default to 100 kbps).
  absl::optional<int> screencast_min_bitrate_kbps;
  // Set by screencast sources. Implies selection of encoding settings
  // suitable for screencast. Most likely not the right way to do
  // things, e.g., screencast of a text document and screencast of a
  // youtube video have different needs.
  absl::optional<bool> is_screencast;
  webrtc::VideoTrackInterface::ContentHint content_hint;

 private:
  template <typename T>
  static void SetFrom(absl::optional<T>* s, const absl::optional<T>& o) {
    if (o) {
      *s = o;
    }
  }
};

class MediaChannelNetworkInterface {
 public:
  enum SocketType { ST_RTP, ST_RTCP };
  virtual bool SendPacket(rtc::CopyOnWriteBuffer* packet,
                          const rtc::PacketOptions& options) = 0;
  virtual bool SendRtcp(rtc::CopyOnWriteBuffer* packet,
                        const rtc::PacketOptions& options) = 0;
  virtual int SetOption(SocketType type,
                        rtc::Socket::Option opt,
                        int option) = 0;
  virtual ~MediaChannelNetworkInterface() {}
};

class MediaSendChannelInterface {
 public:
  virtual ~MediaSendChannelInterface() = default;

  virtual VideoMediaSendChannelInterface* AsVideoSendChannel() = 0;

  virtual VoiceMediaSendChannelInterface* AsVoiceSendChannel() = 0;
  virtual cricket::MediaType media_type() const = 0;

  // Gets the currently set codecs/payload types to be used for outgoing media.
  virtual absl::optional<Codec> GetSendCodec() const = 0;

  // Creates a new outgoing media stream with SSRCs and CNAME as described
  // by sp.
  virtual bool AddSendStream(const StreamParams& sp) = 0;
  // Removes an outgoing media stream.
  // SSRC must be the first SSRC of the media stream if the stream uses
  // multiple SSRCs. In the case of an ssrc of 0, the possibly cached
  // StreamParams is removed.
  virtual bool RemoveSendStream(uint32_t ssrc) = 0;
  // Called on the network thread after a transport has finished sending a
  // packet.
  virtual void OnPacketSent(const rtc::SentPacket& sent_packet) = 0;
  // Called when the socket's ability to send has changed.
  virtual void OnReadyToSend(bool ready) = 0;
  // Called when the network route used for sending packets changed.
  virtual void OnNetworkRouteChanged(
      absl::string_view transport_name,
      const rtc::NetworkRoute& network_route) = 0;
  // Sets the abstract interface class for sending RTP/RTCP data.
  virtual void SetInterface(MediaChannelNetworkInterface* iface) = 0;

  // Returns `true` if a non-null MediaChannelNetworkInterface pointer is held.
  // Must be called on the network thread.
  virtual bool HasNetworkInterface() const = 0;

  // Corresponds to the SDP attribute extmap-allow-mixed, see RFC8285.
  // Set to true if it's allowed to mix one- and two-byte RTP header extensions
  // in the same stream. The setter and getter must only be called from
  // worker_thread.
  virtual void SetExtmapAllowMixed(bool extmap_allow_mixed) = 0;
  virtual bool ExtmapAllowMixed() const = 0;

  // Set the frame encryptor to use on all outgoing frames. This is optional.
  // This pointers lifetime is managed by the set of RtpSender it is attached
  // to.
  virtual void SetFrameEncryptor(
      uint32_t ssrc,
      rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) = 0;

  virtual webrtc::RTCError SetRtpSendParameters(
      uint32_t ssrc,
      const webrtc::RtpParameters& parameters,
      webrtc::SetParametersCallback callback = nullptr) = 0;

  virtual void SetEncoderToPacketizerFrameTransformer(
      uint32_t ssrc,
      rtc::scoped_refptr<webrtc::FrameTransformerInterface>
          frame_transformer) = 0;

  // note: The encoder_selector object must remain valid for the lifetime of the
  // MediaChannel, unless replaced.
  virtual void SetEncoderSelector(
      uint32_t ssrc,
      webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) {
  }
  virtual webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const = 0;
  virtual bool SendCodecHasNack() const = 0;
  // Called whenever the list of sending SSRCs changes.
  virtual void SetSsrcListChangedCallback(
      absl::AnyInvocable<void(const std::set<uint32_t>&)> callback) = 0;
  // TODO(bugs.webrtc.org/13931): Remove when configuration is more sensible
  virtual void SetSendCodecChangedCallback(
      absl::AnyInvocable<void()> callback) = 0;
};

class MediaReceiveChannelInterface {
 public:
  virtual ~MediaReceiveChannelInterface() = default;

  virtual VideoMediaReceiveChannelInterface* AsVideoReceiveChannel() = 0;
  virtual VoiceMediaReceiveChannelInterface* AsVoiceReceiveChannel() = 0;

  virtual cricket::MediaType media_type() const = 0;
  // Creates a new incoming media stream with SSRCs, CNAME as described
  // by sp. In the case of a sp without SSRCs, the unsignaled sp is cached
  // to be used later for unsignaled streams received.
  virtual bool AddRecvStream(const StreamParams& sp) = 0;
  // Removes an incoming media stream.
  // ssrc must be the first SSRC of the media stream if the stream uses
  // multiple SSRCs.
  virtual bool RemoveRecvStream(uint32_t ssrc) = 0;
  // Resets any cached StreamParams for an unsignaled RecvStream, and removes
  // any existing unsignaled streams.
  virtual void ResetUnsignaledRecvStream() = 0;
  // Sets the abstract interface class for sending RTP/RTCP data.
  virtual void SetInterface(MediaChannelNetworkInterface* iface) = 0;
  // Called on the network when an RTP packet is received.
  virtual void OnPacketReceived(const webrtc::RtpPacketReceived& packet) = 0;
  // Gets the current unsignaled receive stream's SSRC, if there is one.
  virtual absl::optional<uint32_t> GetUnsignaledSsrc() const = 0;
  // Sets the local SSRC for listening to incoming RTCP reports.
  virtual void ChooseReceiverReportSsrc(const std::set<uint32_t>& choices) = 0;
  // This is currently a workaround because of the demuxer state being managed
  // across two separate threads. Once the state is consistently managed on
  // the same thread (network), this workaround can be removed.
  // These two notifications inform the media channel when the transport's
  // demuxer criteria is being updated.
  // * OnDemuxerCriteriaUpdatePending() happens on the same thread that the
  //   channel's streams are added and removed (worker thread).
  // * OnDemuxerCriteriaUpdateComplete() happens on the same thread.
  // Because the demuxer is updated asynchronously, there is a window of time
  // where packets are arriving to the channel for streams that have already
  // been removed on the worker thread. It is important NOT to treat these as
  // new unsignalled ssrcs.
  virtual void OnDemuxerCriteriaUpdatePending() = 0;
  virtual void OnDemuxerCriteriaUpdateComplete() = 0;
  // Set the frame decryptor to use on all incoming frames. This is optional.
  // This pointers lifetimes is managed by the set of RtpReceivers it is
  // attached to.
  virtual void SetFrameDecryptor(
      uint32_t ssrc,
      rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) = 0;

  virtual void SetDepacketizerToDecoderFrameTransformer(
      uint32_t ssrc,
      rtc::scoped_refptr<webrtc::FrameTransformerInterface>
          frame_transformer) = 0;

  // Set base minimum delay of the receive stream with specified ssrc.
  // Base minimum delay sets lower bound on minimum delay value which
  // determines minimum delay until audio playout.
  // Returns false if there is no stream with given ssrc.
  virtual bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) = 0;

  // Returns current value of base minimum delay in milliseconds.
  virtual absl::optional<int> GetBaseMinimumPlayoutDelayMs(
      uint32_t ssrc) const = 0;
};

// The stats information is structured as follows:
// Media are represented by either MediaSenderInfo or MediaReceiverInfo.
// Media contains a vector of SSRC infos that are exclusively used by this
// media. (SSRCs shared between media streams can't be represented.)

// Information about an SSRC.
// This data may be locally recorded, or received in an RTCP SR or RR.
struct SsrcSenderInfo {
  uint32_t ssrc = 0;
  double timestamp = 0.0;  // NTP timestamp, represented as seconds since epoch.
};

struct SsrcReceiverInfo {
  uint32_t ssrc = 0;
  double timestamp = 0.0;
};

struct MediaSenderInfo {
  MediaSenderInfo();
  ~MediaSenderInfo();
  void add_ssrc(const SsrcSenderInfo& stat) { local_stats.push_back(stat); }
  // Temporary utility function for call sites that only provide SSRC.
  // As more info is added into SsrcSenderInfo, this function should go away.
  void add_ssrc(uint32_t ssrc) {
    SsrcSenderInfo stat;
    stat.ssrc = ssrc;
    add_ssrc(stat);
  }
  // Utility accessor for clients that are only interested in ssrc numbers.
  std::vector<uint32_t> ssrcs() const {
    std::vector<uint32_t> retval;
    for (std::vector<SsrcSenderInfo>::const_iterator it = local_stats.begin();
         it != local_stats.end(); ++it) {
      retval.push_back(it->ssrc);
    }
    return retval;
  }
  // Returns true if the media has been connected.
  bool connected() const { return local_stats.size() > 0; }
  // Utility accessor for clients that make the assumption only one ssrc
  // exists per media.
  // This will eventually go away.
  // Call sites that compare this to zero should use connected() instead.
  // https://bugs.webrtc.org/8694
  uint32_t ssrc() const {
    if (connected()) {
      return local_stats[0].ssrc;
    } else {
      return 0;
    }
  }
  // https://w3c.github.io/webrtc-stats/#dom-rtcsentrtpstreamstats-bytessent
  int64_t payload_bytes_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-headerbytessent
  int64_t header_and_padding_bytes_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedbytessent
  uint64_t retransmitted_bytes_sent = 0;
  int packets_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedpacketssent
  uint64_t retransmitted_packets_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-nackcount
  uint32_t nacks_received = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-targetbitrate
  absl::optional<double> target_bitrate;
  int packets_lost = 0;
  float fraction_lost = 0.0f;
  int64_t rtt_ms = 0;
  std::string codec_name;
  absl::optional<int> codec_payload_type;
  std::vector<SsrcSenderInfo> local_stats;
  std::vector<SsrcReceiverInfo> remote_stats;
  // A snapshot of the most recent Report Block with additional data of interest
  // to statistics. Used to implement RTCRemoteInboundRtpStreamStats. Within
  // this list, the `ReportBlockData::source_ssrc()`, which is the SSRC of the
  // corresponding outbound RTP stream, is unique.
  std::vector<webrtc::ReportBlockData> report_block_datas;
  absl::optional<bool> active;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalpacketsenddelay
  webrtc::TimeDelta total_packet_send_delay = webrtc::TimeDelta::Zero();
};

struct MediaReceiverInfo {
  MediaReceiverInfo();
  ~MediaReceiverInfo();

  void add_ssrc(const SsrcReceiverInfo& stat) { local_stats.push_back(stat); }
  // Temporary utility function for call sites that only provide SSRC.
  // As more info is added into SsrcSenderInfo, this function should go away.
  void add_ssrc(uint32_t ssrc) {
    SsrcReceiverInfo stat;
    stat.ssrc = ssrc;
    add_ssrc(stat);
  }
  std::vector<uint32_t> ssrcs() const {
    std::vector<uint32_t> retval;
    for (std::vector<SsrcReceiverInfo>::const_iterator it = local_stats.begin();
         it != local_stats.end(); ++it) {
      retval.push_back(it->ssrc);
    }
    return retval;
  }
  // Returns true if the media has been connected.
  bool connected() const { return local_stats.size() > 0; }
  // Utility accessor for clients that make the assumption only one ssrc
  // exists per media.
  // This will eventually go away.
  // Call sites that compare this to zero should use connected();
  // https://bugs.webrtc.org/8694
  uint32_t ssrc() const {
    if (connected()) {
      return local_stats[0].ssrc;
    } else {
      return 0;
    }
  }

  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-bytesreceived
  int64_t payload_bytes_received = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-headerbytesreceived
  int64_t header_and_padding_bytes_received = 0;
  int packets_received = 0;
  int packets_lost = 0;

  absl::optional<uint64_t> retransmitted_bytes_received;
  absl::optional<uint64_t> retransmitted_packets_received;
  absl::optional<uint32_t> nacks_sent;
  // Jitter (network-related) latency (cumulative).
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferdelay
  double jitter_buffer_delay_seconds = 0.0;
  // Target delay for the jitter buffer (cumulative).
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbuffertargetdelay
  double jitter_buffer_target_delay_seconds = 0.0;
  // Minimum obtainable delay for the jitter buffer (cumulative).
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferminimumdelay
  double jitter_buffer_minimum_delay_seconds = 0.0;
  // Number of observations for cumulative jitter latency.
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferemittedcount
  uint64_t jitter_buffer_emitted_count = 0;
  // The timestamp at which the last packet was received, i.e. the time of the
  // local clock when it was received - not the RTP timestamp of that packet.
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-lastpacketreceivedtimestamp
  absl::optional<webrtc::Timestamp> last_packet_received;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-estimatedplayouttimestamp
  absl::optional<int64_t> estimated_playout_ntp_timestamp_ms;
  std::string codec_name;
  absl::optional<int> codec_payload_type;
  std::vector<SsrcReceiverInfo> local_stats;
  std::vector<SsrcSenderInfo> remote_stats;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-fecpacketsreceived
  absl::optional<uint64_t> fec_packets_received;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-fecpacketsdiscarded
  absl::optional<uint64_t> fec_packets_discarded;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-fecbytesreceived
  absl::optional<uint64_t> fec_bytes_received;
  // https://www.w3.org/TR/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay
  double total_processing_delay_seconds = 0.0;

  // Remote outbound stats derived by the received RTCP sender reports.
  // https://w3c.github.io/webrtc-stats/#remoteoutboundrtpstats-dict*
  absl::optional<int64_t> last_sender_report_timestamp_ms;
  absl::optional<int64_t> last_sender_report_remote_timestamp_ms;
  uint64_t sender_reports_packets_sent = 0;
  uint64_t sender_reports_bytes_sent = 0;
  uint64_t sender_reports_reports_count = 0;
  // These require a DLRR block, see
  // https://w3c.github.io/webrtc-stats/#dom-rtcremoteoutboundrtpstreamstats-roundtriptime
  absl::optional<webrtc::TimeDelta> round_trip_time;
  webrtc::TimeDelta total_round_trip_time = webrtc::TimeDelta::Zero();
  int round_trip_time_measurements = 0;
};

struct VoiceSenderInfo : public MediaSenderInfo {
  VoiceSenderInfo();
  ~VoiceSenderInfo();
  int jitter_ms = 0;
  // Current audio level, expressed linearly [0,32767].
  int audio_level = 0;
  // See description of "totalAudioEnergy" in the WebRTC stats spec:
  // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
  double total_input_energy = 0.0;
  double total_input_duration = 0.0;
  webrtc::ANAStats ana_statistics;
  webrtc::AudioProcessingStats apm_statistics;
};

struct VoiceReceiverInfo : public MediaReceiverInfo {
  VoiceReceiverInfo();
  ~VoiceReceiverInfo();
  int jitter_ms = 0;
  int jitter_buffer_ms = 0;
  int jitter_buffer_preferred_ms = 0;
  int delay_estimate_ms = 0;
  int audio_level = 0;
  // Stats below correspond to similarly-named fields in the WebRTC stats spec.
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats
  double total_output_energy = 0.0;
  uint64_t total_samples_received = 0;
  double total_output_duration = 0.0;
  uint64_t concealed_samples = 0;
  uint64_t silent_concealed_samples = 0;
  uint64_t concealment_events = 0;
  uint64_t inserted_samples_for_deceleration = 0;
  uint64_t removed_samples_for_acceleration = 0;
  // Stats below correspond to similarly-named fields in the WebRTC stats spec.
  // https://w3c.github.io/webrtc-stats/#dom-rtcreceivedrtpstreamstats
  uint64_t packets_discarded = 0;
  // Stats below DO NOT correspond directly to anything in the WebRTC stats
  // fraction of synthesized audio inserted through expansion.
  float expand_rate = 0.0f;
  // fraction of synthesized speech inserted through expansion.
  float speech_expand_rate = 0.0f;
  // fraction of data out of secondary decoding, including FEC and RED.
  float secondary_decoded_rate = 0.0f;
  // Fraction of secondary data, including FEC and RED, that is discarded.
  // Discarding of secondary data can be caused by the reception of the primary
  // data, obsoleting the secondary data. It can also be caused by early
  // or late arrival of secondary data. This metric is the percentage of
  // discarded secondary data since last query of receiver info.
  float secondary_discarded_rate = 0.0f;
  // Fraction of data removed through time compression.
  float accelerate_rate = 0.0f;
  // Fraction of data inserted through time stretching.
  float preemptive_expand_rate = 0.0f;
  int decoding_calls_to_silence_generator = 0;
  int decoding_calls_to_neteq = 0;
  int decoding_normal = 0;
  // TODO(alexnarest): Consider decoding_neteq_plc for consistency
  int decoding_plc = 0;
  int decoding_codec_plc = 0;
  int decoding_cng = 0;
  int decoding_plc_cng = 0;
  int decoding_muted_output = 0;
  // Estimated capture start time in NTP time in ms.
  int64_t capture_start_ntp_time_ms = -1;
  // Count of the number of buffer flushes.
  uint64_t jitter_buffer_flushes = 0;
  // Number of samples expanded due to delayed packets.
  uint64_t delayed_packet_outage_samples = 0;
  // Arrival delay of received audio packets.
  double relative_packet_arrival_delay_seconds = 0.0;
  // Count and total duration of audio interruptions (loss-concealement periods
  // longer than 150 ms).
  int32_t interruption_count = 0;
  int32_t total_interruption_duration_ms = 0;
};

struct VideoSenderInfo : public MediaSenderInfo {
  VideoSenderInfo();
  ~VideoSenderInfo();
  std::vector<SsrcGroup> ssrc_groups;
  absl::optional<std::string> encoder_implementation_name;
  int firs_received = 0;
  int plis_received = 0;
  int send_frame_width = 0;
  int send_frame_height = 0;
  int frames = 0;
  double framerate_input = 0;
  int framerate_sent = 0;
  int aggregated_framerate_sent = 0;
  int nominal_bitrate = 0;
  int adapt_reason = 0;
  int adapt_changes = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationreason
  webrtc::QualityLimitationReason quality_limitation_reason =
      webrtc::QualityLimitationReason::kNone;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationdurations
  std::map<webrtc::QualityLimitationReason, int64_t>
      quality_limitation_durations_ms;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationresolutionchanges
  uint32_t quality_limitation_resolution_changes = 0;
  int avg_encode_ms = 0;
  int encode_usage_percent = 0;
  uint32_t frames_encoded = 0;
  uint32_t key_frames_encoded = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodetime
  uint64_t total_encode_time_ms = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget
  uint64_t total_encoded_bytes_target = 0;
  bool has_entered_low_resolution = false;
  absl::optional<uint64_t> qp_sum;
  webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;
  uint32_t frames_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcvideosenderstats-hugeframessent
  uint32_t huge_frames_sent = 0;
  uint32_t aggregated_huge_frames_sent = 0;
  absl::optional<std::string> rid;
  absl::optional<bool> power_efficient_encoder;
  absl::optional<webrtc::ScalabilityMode> scalability_mode;
};

struct VideoReceiverInfo : public MediaReceiverInfo {
  VideoReceiverInfo();
  ~VideoReceiverInfo();
  std::vector<SsrcGroup> ssrc_groups;
  absl::optional<std::string> decoder_implementation_name;
  absl::optional<bool> power_efficient_decoder;
  int packets_concealed = 0;
  int firs_sent = 0;
  int plis_sent = 0;
  int frame_width = 0;
  int frame_height = 0;
  int framerate_received = 0;
  int framerate_decoded = 0;
  int framerate_output = 0;
  // Framerate as sent to the renderer.
  int framerate_render_input = 0;
  // Framerate that the renderer reports.
  int framerate_render_output = 0;
  uint32_t frames_received = 0;
  uint32_t frames_dropped = 0;
  uint32_t frames_decoded = 0;
  uint32_t key_frames_decoded = 0;
  uint32_t frames_rendered = 0;
  absl::optional<uint64_t> qp_sum;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime
  webrtc::TimeDelta total_decode_time = webrtc::TimeDelta::Zero();
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay
  webrtc::TimeDelta total_processing_delay = webrtc::TimeDelta::Zero();
  webrtc::TimeDelta total_assembly_time = webrtc::TimeDelta::Zero();
  uint32_t frames_assembled_from_multiple_packets = 0;
  double total_inter_frame_delay = 0;
  double total_squared_inter_frame_delay = 0;
  int64_t interframe_delay_max_ms = -1;
  uint32_t freeze_count = 0;
  uint32_t pause_count = 0;
  uint32_t total_freezes_duration_ms = 0;
  uint32_t total_pauses_duration_ms = 0;
  uint32_t jitter_ms = 0;

  webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;

  // All stats below are gathered per-VideoReceiver, but some will be correlated
  // across MediaStreamTracks.  NOTE(hta): when sinking stats into per-SSRC
  // structures, reflect this in the new layout.

  // Current frame decode latency.
  int decode_ms = 0;
  // Maximum observed frame decode latency.
  int max_decode_ms = 0;
  // Jitter (network-related) latency.
  int jitter_buffer_ms = 0;
  // Requested minimum playout latency.
  int min_playout_delay_ms = 0;
  // Requested latency to account for rendering delay.
  int render_delay_ms = 0;
  // Target overall delay: network+decode+render, accounting for
  // min_playout_delay_ms.
  int target_delay_ms = 0;
  // Current overall delay, possibly ramping towards target_delay_ms.
  int current_delay_ms = 0;

  // Estimated capture start time in NTP time in ms.
  int64_t capture_start_ntp_time_ms = -1;

  // First frame received to first frame decoded latency.
  int64_t first_frame_received_to_decoded_ms = -1;

  // Timing frame info: all important timestamps for a full lifetime of a
  // single 'timing frame'.
  absl::optional<webrtc::TimingFrameInfo> timing_frame_info;
};

struct BandwidthEstimationInfo {
  int available_send_bandwidth = 0;
  int available_recv_bandwidth = 0;
  int target_enc_bitrate = 0;
  int actual_enc_bitrate = 0;
  int retransmit_bitrate = 0;
  int transmit_bitrate = 0;
  int64_t bucket_delay = 0;
};

// Maps from payload type to `RtpCodecParameters`.
typedef std::map<int, webrtc::RtpCodecParameters> RtpCodecParametersMap;

// Stats returned from VoiceMediaSendChannel.GetStats()
struct VoiceMediaSendInfo {
  VoiceMediaSendInfo();
  ~VoiceMediaSendInfo();
  void Clear() {
    senders.clear();
    send_codecs.clear();
  }
  std::vector<VoiceSenderInfo> senders;
  RtpCodecParametersMap send_codecs;
};

// Stats returned from VoiceMediaReceiveChannel.GetStats()
struct VoiceMediaReceiveInfo {
  VoiceMediaReceiveInfo();
  ~VoiceMediaReceiveInfo();
  void Clear() {
    receivers.clear();
    receive_codecs.clear();
  }
  std::vector<VoiceReceiverInfo> receivers;
  RtpCodecParametersMap receive_codecs;
  int32_t device_underrun_count = 0;
};

// Combined VoiceMediaSendInfo and VoiceMediaReceiveInfo
// Returned from Transceiver.getStats()
struct VoiceMediaInfo {
  VoiceMediaInfo();
  VoiceMediaInfo(VoiceMediaSendInfo&& send, VoiceMediaReceiveInfo&& receive)
      : senders(std::move(send.senders)),
        receivers(std::move(receive.receivers)),
        send_codecs(std::move(send.send_codecs)),
        receive_codecs(std::move(receive.receive_codecs)),
        device_underrun_count(receive.device_underrun_count) {}
  ~VoiceMediaInfo();
  void Clear() {
    senders.clear();
    receivers.clear();
    send_codecs.clear();
    receive_codecs.clear();
  }
  std::vector<VoiceSenderInfo> senders;
  std::vector<VoiceReceiverInfo> receivers;
  RtpCodecParametersMap send_codecs;
  RtpCodecParametersMap receive_codecs;
  int32_t device_underrun_count = 0;
};

// Stats for a VideoMediaSendChannel
struct VideoMediaSendInfo {
  VideoMediaSendInfo();
  ~VideoMediaSendInfo();
  void Clear() {
    senders.clear();
    aggregated_senders.clear();
    send_codecs.clear();
  }
  // Each sender info represents one "outbound-rtp" stream.In non - simulcast,
  // this means one info per RtpSender but if simulcast is used this means
  // one info per simulcast layer.
  std::vector<VideoSenderInfo> senders;
  // Used for legacy getStats() API's "ssrc" stats and modern getStats() API's
  // "track" stats. If simulcast is used, instead of having one sender info per
  // simulcast layer, the metrics of all layers of an RtpSender are aggregated
  // into a single sender info per RtpSender.
  std::vector<VideoSenderInfo> aggregated_senders;
  RtpCodecParametersMap send_codecs;
};

// Stats for a VideoMediaReceiveChannel
struct VideoMediaReceiveInfo {
  VideoMediaReceiveInfo();
  ~VideoMediaReceiveInfo();
  void Clear() {
    receivers.clear();
    receive_codecs.clear();
  }
  std::vector<VideoReceiverInfo> receivers;
  RtpCodecParametersMap receive_codecs;
};

// Combined VideoMediaSenderInfo and VideoMediaReceiverInfo.
// Returned from channel.GetStats()
struct VideoMediaInfo {
  VideoMediaInfo();
  VideoMediaInfo(VideoMediaSendInfo&& send, VideoMediaReceiveInfo&& receive)
      : senders(std::move(send.senders)),
        aggregated_senders(std::move(send.aggregated_senders)),
        receivers(std::move(receive.receivers)),
        send_codecs(std::move(send.send_codecs)),
        receive_codecs(std::move(receive.receive_codecs)) {}
  ~VideoMediaInfo();
  void Clear() {
    senders.clear();
    aggregated_senders.clear();
    receivers.clear();
    send_codecs.clear();
    receive_codecs.clear();
  }
  // Each sender info represents one "outbound-rtp" stream. In non-simulcast,
  // this means one info per RtpSender but if simulcast is used this means
  // one info per simulcast layer.
  std::vector<VideoSenderInfo> senders;
  // Used for legacy getStats() API's "ssrc" stats and modern getStats() API's
  // "track" stats. If simulcast is used, instead of having one sender info per
  // simulcast layer, the metrics of all layers of an RtpSender are aggregated
  // into a single sender info per RtpSender.
  std::vector<VideoSenderInfo> aggregated_senders;
  std::vector<VideoReceiverInfo> receivers;
  RtpCodecParametersMap send_codecs;
  RtpCodecParametersMap receive_codecs;
};

struct RtcpParameters {
  bool reduced_size = false;
  bool remote_estimate = false;
};

struct MediaChannelParameters {
  virtual ~MediaChannelParameters() = default;

  std::vector<Codec> codecs;
  std::vector<webrtc::RtpExtension> extensions;
  // For a send stream this is true if we've negotiated a send direction,
  // for a receive stream this is true if we've negotiated a receive direction.
  bool is_stream_active = true;

  // TODO(pthatcher): Add streams.
  RtcpParameters rtcp;

  std::string ToString() const {
    rtc::StringBuilder ost;
    ost << "{";
    const char* separator = "";
    for (const auto& entry : ToStringMap()) {
      ost << separator << entry.first << ": " << entry.second;
      separator = ", ";
    }
    ost << "}";
    return ost.Release();
  }

 protected:
  virtual std::map<std::string, std::string> ToStringMap() const {
    return {{"codecs", VectorToString(codecs)},
            {"extensions", VectorToString(extensions)},
            {"rtcp", "{reduced_size:" + rtc::ToString(rtcp.reduced_size) +
                         ", remote_estimate:" +
                         rtc::ToString(rtcp.remote_estimate) + "}"}};
  }
};

struct SenderParameters : MediaChannelParameters {
  int max_bandwidth_bps = -1;
  // This is the value to be sent in the MID RTP header extension (if the header
  // extension in included in the list of extensions).
  std::string mid;
  bool extmap_allow_mixed = false;

 protected:
  std::map<std::string, std::string> ToStringMap() const override {
    auto params = MediaChannelParameters::ToStringMap();
    params["max_bandwidth_bps"] = rtc::ToString(max_bandwidth_bps);
    params["mid"] = (mid.empty() ? "<not set>" : mid);
    params["extmap-allow-mixed"] = extmap_allow_mixed ? "true" : "false";
    return params;
  }
};

struct AudioSenderParameter : SenderParameters {
  AudioSenderParameter();
  ~AudioSenderParameter() override;
  AudioOptions options;

 protected:
  std::map<std::string, std::string> ToStringMap() const override;
};

struct AudioReceiverParameters : MediaChannelParameters {};

class VoiceMediaSendChannelInterface : public MediaSendChannelInterface {
 public:
  virtual bool SetSenderParameters(const AudioSenderParameter& params) = 0;
  // Starts or stops sending (and potentially capture) of local audio.
  virtual void SetSend(bool send) = 0;
  // Configure stream for sending.
  virtual bool SetAudioSend(uint32_t ssrc,
                            bool enable,
                            const AudioOptions* options,
                            AudioSource* source) = 0;
  // Returns if the telephone-event has been negotiated.
  virtual bool CanInsertDtmf() = 0;
  // Send a DTMF `event`. The DTMF out-of-band signal will be used.
  // The `ssrc` should be either 0 or a valid send stream ssrc.
  // The valid value for the `event` are 0 to 15 which corresponding to
  // DTMF event 0-9, *, #, A-D.
  virtual bool InsertDtmf(uint32_t ssrc, int event, int duration) = 0;
  virtual bool GetStats(VoiceMediaSendInfo* stats) = 0;
  virtual bool SenderNackEnabled() const = 0;
  virtual bool SenderNonSenderRttEnabled() const = 0;
};

class VoiceMediaReceiveChannelInterface : public MediaReceiveChannelInterface {
 public:
  virtual bool SetReceiverParameters(const AudioReceiverParameters& params) = 0;
  // Get the receive parameters for the incoming stream identified by `ssrc`.
  virtual webrtc::RtpParameters GetRtpReceiverParameters(
      uint32_t ssrc) const = 0;
  virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
  // Retrieve the receive parameters for the default receive
  // stream, which is used when SSRCs are not signaled.
  virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0;
  // Starts or stops playout of received audio.
  virtual void SetPlayout(bool playout) = 0;
  // Set speaker output volume of the specified ssrc.
  virtual bool SetOutputVolume(uint32_t ssrc, double volume) = 0;
  // Set speaker output volume for future unsignaled streams.
  virtual bool SetDefaultOutputVolume(double volume) = 0;
  virtual void SetRawAudioSink(
      uint32_t ssrc,
      std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0;
  virtual void SetDefaultRawAudioSink(
      std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0;
  virtual bool GetStats(VoiceMediaReceiveInfo* stats, bool reset_legacy) = 0;
  virtual void SetReceiveNackEnabled(bool enabled) = 0;
  virtual void SetRtcpMode(webrtc::RtcpMode mode) = 0;
  virtual void SetReceiveNonSenderRttEnabled(bool enabled) = 0;
};

struct VideoSenderParameters : SenderParameters {
  VideoSenderParameters();
  ~VideoSenderParameters() override;
  // Use conference mode? This flag comes from the remote
  // description's SDP line 'a=x-google-flag:conference', copied over
  // by VideoChannel::SetRemoteContent_w, and ultimately used by
  // conference mode screencast logic in
  // WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig.
  // The special screencast behaviour is disabled by default.
  bool conference_mode = false;

 protected:
  std::map<std::string, std::string> ToStringMap() const override;
};

struct VideoReceiverParameters : MediaChannelParameters {};

class VideoMediaSendChannelInterface : public MediaSendChannelInterface {
 public:
  virtual bool SetSenderParameters(const VideoSenderParameters& params) = 0;
  // Starts or stops transmission (and potentially capture) of local video.
  virtual bool SetSend(bool send) = 0;
  // Configure stream for sending and register a source.
  // The `ssrc` must correspond to a registered send stream.
  virtual bool SetVideoSend(
      uint32_t ssrc,
      const VideoOptions* options,
      rtc::VideoSourceInterface<webrtc::VideoFrame>* source) = 0;
  // Cause generation of a keyframe for `ssrc` on a sending channel.
  virtual void GenerateSendKeyFrame(uint32_t ssrc,
                                    const std::vector<std::string>& rids) = 0;
  virtual bool GetStats(VideoMediaSendInfo* stats) = 0;
  // This fills the "bitrate parts" (rtx, video bitrate) of the
  // BandwidthEstimationInfo, since that part that isn't possible to get
  // through webrtc::Call::GetStats, as they are statistics of the send
  // streams.
  // TODO(holmer): We should change this so that either BWE graphs doesn't
  // need access to bitrates of the streams, or change the (RTC)StatsCollector
  // so that it's getting the send stream stats separately by calling
  // GetStats(), and merges with BandwidthEstimationInfo by itself.
  virtual void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) = 0;
  // Information queries to support SetReceiverFeedbackParameters
  virtual webrtc::RtcpMode SendCodecRtcpMode() const = 0;
  virtual bool SendCodecHasLntf() const = 0;
  virtual absl::optional<int> SendCodecRtxTime() const = 0;
};

class VideoMediaReceiveChannelInterface : public MediaReceiveChannelInterface {
 public:
  virtual bool SetReceiverParameters(const VideoReceiverParameters& params) = 0;
  // Get the receive parameters for the incoming stream identified by `ssrc`.
  virtual webrtc::RtpParameters GetRtpReceiverParameters(
      uint32_t ssrc) const = 0;
  // Starts or stops decoding of remote video.
  virtual void SetReceive(bool receive) = 0;
  // Retrieve the receive parameters for the default receive
  // stream, which is used when SSRCs are not signaled.
  virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0;
  // Sets the sink object to be used for the specified stream.
  virtual bool SetSink(uint32_t ssrc,
                       rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0;
  // The sink is used for the 'default' stream.
  virtual void SetDefaultSink(
      rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0;
  // Request generation of a keyframe for `ssrc` on a receiving channel via
  // RTCP feedback.
  virtual void RequestRecvKeyFrame(uint32_t ssrc) = 0;

  virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
  // Set recordable encoded frame callback for `ssrc`
  virtual void SetRecordableEncodedFrameCallback(
      uint32_t ssrc,
      std::function<void(const webrtc::RecordableEncodedFrame&)> callback) = 0;
  // Clear recordable encoded frame callback for `ssrc`
  virtual void ClearRecordableEncodedFrameCallback(uint32_t ssrc) = 0;
  virtual bool GetStats(VideoMediaReceiveInfo* stats) = 0;
  virtual void SetReceiverFeedbackParameters(bool lntf_enabled,
                                             bool nack_enabled,
                                             webrtc::RtcpMode rtcp_mode,
                                             absl::optional<int> rtx_time) = 0;
  virtual bool AddDefaultRecvStreamForTesting(const StreamParams& sp) = 0;
};

}  // namespace cricket

#endif  // MEDIA_BASE_MEDIA_CHANNEL_H_