sdk/android/api/org/webrtc/VideoEncoder.java

/*
 *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

package org.webrtc;

import androidx.annotation.Nullable;
import org.webrtc.EncodedImage;

/**
 * Interface for a video encoder that can be used with WebRTC. All calls will be made on the
 * encoding thread. The encoder may be constructed on a different thread and changing thread after
 * calling release is allowed.
 */
public interface VideoEncoder {
  /** Settings passed to the encoder by WebRTC. */
  public class Settings {
    public final int numberOfCores;
    public final int width;
    public final int height;
    public final int startBitrate; // Kilobits per second.
    public final int maxFramerate;
    public final int numberOfSimulcastStreams;
    public final boolean automaticResizeOn;
    public final Capabilities capabilities;

    // TODO(bugs.webrtc.org/10720): Remove.
    @Deprecated
    public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate,
        int numberOfSimulcastStreams, boolean automaticResizeOn) {
      this(numberOfCores, width, height, startBitrate, maxFramerate, numberOfSimulcastStreams,
          automaticResizeOn, new VideoEncoder.Capabilities(false /* lossNotification */));
    }

    @CalledByNative("Settings")
    public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate,
        int numberOfSimulcastStreams, boolean automaticResizeOn, Capabilities capabilities) {
      this.numberOfCores = numberOfCores;
      this.width = width;
      this.height = height;
      this.startBitrate = startBitrate;
      this.maxFramerate = maxFramerate;
      this.numberOfSimulcastStreams = numberOfSimulcastStreams;
      this.automaticResizeOn = automaticResizeOn;
      this.capabilities = capabilities;
    }
  }

  /** Capabilities (loss notification, etc.) passed to the encoder by WebRTC. */
  public class Capabilities {
    /**
     * The remote side has support for the loss notification RTCP feedback message format, and will
     * be sending these feedback messages if necessary.
     */
    public final boolean lossNotification;

    @CalledByNative("Capabilities")
    public Capabilities(boolean lossNotification) {
      this.lossNotification = lossNotification;
    }
  }

  /** Additional info for encoding. */
  public class EncodeInfo {
    public final EncodedImage.FrameType[] frameTypes;

    @CalledByNative("EncodeInfo")
    public EncodeInfo(EncodedImage.FrameType[] frameTypes) {
      this.frameTypes = frameTypes;
    }
  }

  // TODO(sakal): Add values to these classes as necessary.
  /** Codec specific information about the encoded frame. */
  public class CodecSpecificInfo {}

  public class CodecSpecificInfoVP8 extends CodecSpecificInfo {}

  public class CodecSpecificInfoVP9 extends CodecSpecificInfo {}

  public class CodecSpecificInfoH264 extends CodecSpecificInfo {}

  public class CodecSpecificInfoAV1 extends CodecSpecificInfo {}

  /**
   * Represents bitrate allocated for an encoder to produce frames. Bitrate can be divided between
   * spatial and temporal layers.
   */
  public class BitrateAllocation {
    // First index is the spatial layer and second the temporal layer.
    public final int[][] bitratesBbs;

    /**
     * Initializes the allocation with a two dimensional array of bitrates. The first index of the
     * array is the spatial layer and the second index in the temporal layer.
     */
    @CalledByNative("BitrateAllocation")
    public BitrateAllocation(int[][] bitratesBbs) {
      this.bitratesBbs = bitratesBbs;
    }

    /**
     * Gets the total bitrate allocated for all layers.
     */
    public int getSum() {
      int sum = 0;
      for (int[] spatialLayer : bitratesBbs) {
        for (int bitrate : spatialLayer) {
          sum += bitrate;
        }
      }
      return sum;
    }
  }

  /** Settings for WebRTC quality based scaling. */
  public class ScalingSettings {
    public final boolean on;
    @Nullable public final Integer low;
    @Nullable public final Integer high;

    /**
     * Settings to disable quality based scaling.
     */
    public static final ScalingSettings OFF = new ScalingSettings();

    /**
     * Creates settings to enable quality based scaling.
     *
     * @param low Average QP at which to scale up the resolution.
     * @param high Average QP at which to scale down the resolution.
     */
    public ScalingSettings(int low, int high) {
      this.on = true;
      this.low = low;
      this.high = high;
    }

    private ScalingSettings() {
      this.on = false;
      this.low = null;
      this.high = null;
    }

    // TODO(bugs.webrtc.org/8830): Below constructors are deprecated.
    // Default thresholds are going away, so thresholds have to be set
    // when scaling is on.
    /**
     * Creates quality based scaling setting.
     *
     * @param on True if quality scaling is turned on.
     */
    @Deprecated
    public ScalingSettings(boolean on) {
      this.on = on;
      this.low = null;
      this.high = null;
    }

    /**
     * Creates quality based scaling settings with custom thresholds.
     *
     * @param on True if quality scaling is turned on.
     * @param low Average QP at which to scale up the resolution.
     * @param high Average QP at which to scale down the resolution.
     */
    @Deprecated
    public ScalingSettings(boolean on, int low, int high) {
      this.on = on;
      this.low = low;
      this.high = high;
    }

    @Override
    public String toString() {
      return on ? "[ " + low + ", " + high + " ]" : "OFF";
    }
  }

  /**
   * Bitrate limits for resolution.
   */
  public class ResolutionBitrateLimits {
    /**
     * Maximum size of video frame, in pixels, the bitrate limits are intended for.
     */
    public final int frameSizePixels;

    /**
     * Recommended minimum bitrate to start encoding.
     */
    public final int minStartBitrateBps;

    /**
     * Recommended minimum bitrate.
     */
    public final int minBitrateBps;

    /**
     * Recommended maximum bitrate.
     */
    public final int maxBitrateBps;

    public ResolutionBitrateLimits(
        int frameSizePixels, int minStartBitrateBps, int minBitrateBps, int maxBitrateBps) {
      this.frameSizePixels = frameSizePixels;
      this.minStartBitrateBps = minStartBitrateBps;
      this.minBitrateBps = minBitrateBps;
      this.maxBitrateBps = maxBitrateBps;
    }

    @CalledByNative("ResolutionBitrateLimits")
    public int getFrameSizePixels() {
      return frameSizePixels;
    }

    @CalledByNative("ResolutionBitrateLimits")
    public int getMinStartBitrateBps() {
      return minStartBitrateBps;
    }

    @CalledByNative("ResolutionBitrateLimits")
    public int getMinBitrateBps() {
      return minBitrateBps;
    }

    @CalledByNative("ResolutionBitrateLimits")
    public int getMaxBitrateBps() {
      return maxBitrateBps;
    }
  }

  /** Rate control parameters. */
  public class RateControlParameters {
    /**
     * Adjusted target bitrate, per spatial/temporal layer. May be lower or higher than the target
     * depending on encoder behaviour.
     */
    public final BitrateAllocation bitrate;

    /**
     * Target framerate, in fps. A value <= 0.0 is invalid and should be interpreted as framerate
     * target not available. In this case the encoder should fall back to the max framerate
     * specified in `codec_settings` of the last InitEncode() call.
     */
    public final double framerateFps;

    @CalledByNative("RateControlParameters")
    public RateControlParameters(BitrateAllocation bitrate, double framerateFps) {
      this.bitrate = bitrate;
      this.framerateFps = framerateFps;
    }
  }

  /**
   * Metadata about the Encoder.
   */
  public class EncoderInfo {
    /**
     * The width and height of the incoming video frames should be divisible by
     * |requested_resolution_alignment|
     */
    public final int requestedResolutionAlignment;

    /**
     * Same as above but if true, each simulcast layer should also be divisible by
     * |requested_resolution_alignment|.
     */
    public final boolean applyAlignmentToAllSimulcastLayers;

    public EncoderInfo(
        int requestedResolutionAlignment, boolean applyAlignmentToAllSimulcastLayers) {
      this.requestedResolutionAlignment = requestedResolutionAlignment;
      this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers;
    }

    @CalledByNative("EncoderInfo")
    public int getRequestedResolutionAlignment() {
      return requestedResolutionAlignment;
    }

    @CalledByNative("EncoderInfo")
    public boolean getApplyAlignmentToAllSimulcastLayers() {
      return applyAlignmentToAllSimulcastLayers;
    }
  }

  public interface Callback {
    /**
     * Old encoders assume that the byte buffer held by `frame` is not accessed after the call to
     * this method returns. If the pipeline downstream needs to hold on to the buffer, it then has
     * to make its own copy. We want to move to a model where no copying is needed, and instead use
     * retain()/release() to signal to the encoder when it is safe to reuse the buffer.
     *
     * Over the transition, implementations of this class should use the maybeRetain() method if
     * they want to keep a reference to the buffer, and fall back to copying if that method returns
     * false.
     */
    void onEncodedFrame(EncodedImage frame, CodecSpecificInfo info);
  }

  /**
   * The encoder implementation backing this interface is either 1) a Java
   * encoder (e.g., an Android platform encoder), or alternatively 2) a native
   * encoder (e.g., a software encoder or a C++ encoder adapter).
   *
   * For case 1), createNativeVideoEncoder() should return zero.
   * In this case, we expect the native library to call the encoder through
   * JNI using the Java interface declared below.
   *
   * For case 2), createNativeVideoEncoder() should return a non-zero value.
   * In this case, we expect the native library to treat the returned value as
   * a raw pointer of type webrtc::VideoEncoder* (ownership is transferred to
   * the caller). The native library should then directly call the
   * webrtc::VideoEncoder interface without going through JNI. All calls to
   * the Java interface methods declared below should thus throw an
   * UnsupportedOperationException.
   */
  @CalledByNative
  default long createNativeVideoEncoder() {
    return 0;
  }

  /**
   * Returns true if the encoder is backed by hardware.
   */
  @CalledByNative
  default boolean isHardwareEncoder() {
    return true;
  }

  /**
   * Initializes the encoding process. Call before any calls to encode.
   */
  @CalledByNative VideoCodecStatus initEncode(Settings settings, Callback encodeCallback);

  /**
   * Releases the encoder. No more calls to encode will be made after this call.
   */
  @CalledByNative VideoCodecStatus release();

  /**
   * Requests the encoder to encode a frame.
   */
  @CalledByNative VideoCodecStatus encode(VideoFrame frame, EncodeInfo info);

  /** Sets the bitrate allocation and the target framerate for the encoder. */
  VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate);

  /** Sets the bitrate allocation and the target framerate for the encoder. */
  default @CalledByNative VideoCodecStatus setRates(RateControlParameters rcParameters) {
    // Round frame rate up to avoid overshoots.
    int framerateFps = (int) Math.ceil(rcParameters.framerateFps);
    return setRateAllocation(rcParameters.bitrate, framerateFps);
  }

  /** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */
  @CalledByNative ScalingSettings getScalingSettings();

  /** Returns the list of bitrate limits. */
  @CalledByNative
  default ResolutionBitrateLimits[] getResolutionBitrateLimits() {
    // TODO(ssilkin): Update downstream projects and remove default implementation.
    ResolutionBitrateLimits bitrate_limits[] = {};
    return bitrate_limits;
  }

  /**
   * Should return a descriptive name for the implementation. Gets called once and cached. May be
   * called from arbitrary thread.
   */
  @CalledByNative String getImplementationName();

  @CalledByNative
  default EncoderInfo getEncoderInfo() {
    return new EncoderInfo(
        /* requestedResolutionAlignment= */ 1, /* applyAlignmentToAllSimulcastLayers= */ false);
  }
}