sdk/android/api/org/webrtc/VideoEncoder.java - src.git - Git at Google

 /*
  *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */

 package org.webrtc;

 import androidx.annotation.Nullable;
 import org.webrtc.EncodedImage;

 /**
  * Interface for a video encoder that can be used with WebRTC. All calls will be made on the
  * encoding thread. The encoder may be constructed on a different thread and changing thread after
  * calling release is allowed.
  */
 public interface VideoEncoder {
   /** Settings passed to the encoder by WebRTC. */
   public class Settings {
     public final int numberOfCores;
     public final int width;
     public final int height;
     public final int startBitrate; // Kilobits per second.
     public final int maxFramerate;
     public final int numberOfSimulcastStreams;
     public final boolean automaticResizeOn;
     public final Capabilities capabilities;

     // TODO(bugs.webrtc.org/10720): Remove.
     @Deprecated
     public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate,
         int numberOfSimulcastStreams, boolean automaticResizeOn) {
       this(numberOfCores, width, height, startBitrate, maxFramerate, numberOfSimulcastStreams,
           automaticResizeOn, new VideoEncoder.Capabilities(false /* lossNotification */));
     }

     @CalledByNative("Settings")
     public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate,
         int numberOfSimulcastStreams, boolean automaticResizeOn, Capabilities capabilities) {
       this.numberOfCores = numberOfCores;
       this.width = width;
       this.height = height;
       this.startBitrate = startBitrate;
       this.maxFramerate = maxFramerate;
       this.numberOfSimulcastStreams = numberOfSimulcastStreams;
       this.automaticResizeOn = automaticResizeOn;
       this.capabilities = capabilities;
     }
   }

   /** Capabilities (loss notification, etc.) passed to the encoder by WebRTC. */
   public class Capabilities {
     /**
      * The remote side has support for the loss notification RTCP feedback message format, and will
      * be sending these feedback messages if necessary.
      */
     public final boolean lossNotification;

     @CalledByNative("Capabilities")
     public Capabilities(boolean lossNotification) {
       this.lossNotification = lossNotification;
     }
   }

   /** Additional info for encoding. */
   public class EncodeInfo {
     public final EncodedImage.FrameType[] frameTypes;

     @CalledByNative("EncodeInfo")
     public EncodeInfo(EncodedImage.FrameType[] frameTypes) {
       this.frameTypes = frameTypes;
     }
   }

   // TODO(sakal): Add values to these classes as necessary.
   /** Codec specific information about the encoded frame. */
   public class CodecSpecificInfo {}

   public class CodecSpecificInfoVP8 extends CodecSpecificInfo {}

   public class CodecSpecificInfoVP9 extends CodecSpecificInfo {}

   public class CodecSpecificInfoH264 extends CodecSpecificInfo {}

   public class CodecSpecificInfoAV1 extends CodecSpecificInfo {}

   /**
    * Represents bitrate allocated for an encoder to produce frames. Bitrate can be divided between
    * spatial and temporal layers.
    */
   public class BitrateAllocation {
     // First index is the spatial layer and second the temporal layer.
     public final int[][] bitratesBbs;

     /**
      * Initializes the allocation with a two dimensional array of bitrates. The first index of the
      * array is the spatial layer and the second index in the temporal layer.
      */
     @CalledByNative("BitrateAllocation")
     public BitrateAllocation(int[][] bitratesBbs) {
       this.bitratesBbs = bitratesBbs;
     }

     /**
      * Gets the total bitrate allocated for all layers.
      */
     public int getSum() {
       int sum = 0;
       for (int[] spatialLayer : bitratesBbs) {
         for (int bitrate : spatialLayer) {
           sum += bitrate;
         }
       }
       return sum;
     }
   }

   /** Settings for WebRTC quality based scaling. */
   public class ScalingSettings {
     public final boolean on;
     @Nullable public final Integer low;
     @Nullable public final Integer high;

     /**
      * Settings to disable quality based scaling.
      */
     public static final ScalingSettings OFF = new ScalingSettings();

     /**
      * Creates settings to enable quality based scaling.
      *
      * @param low Average QP at which to scale up the resolution.
      * @param high Average QP at which to scale down the resolution.
      */
     public ScalingSettings(int low, int high) {
       this.on = true;
       this.low = low;
       this.high = high;
     }

     private ScalingSettings() {
       this.on = false;
       this.low = null;
       this.high = null;
     }

     // TODO(bugs.webrtc.org/8830): Below constructors are deprecated.
     // Default thresholds are going away, so thresholds have to be set
     // when scaling is on.
     /**
      * Creates quality based scaling setting.
      *
      * @param on True if quality scaling is turned on.
      */
     @Deprecated
     public ScalingSettings(boolean on) {
       this.on = on;
       this.low = null;
       this.high = null;
     }

     /**
      * Creates quality based scaling settings with custom thresholds.
      *
      * @param on True if quality scaling is turned on.
      * @param low Average QP at which to scale up the resolution.
      * @param high Average QP at which to scale down the resolution.
      */
     @Deprecated
     public ScalingSettings(boolean on, int low, int high) {
       this.on = on;
       this.low = low;
       this.high = high;
     }

     @Override
     public String toString() {
       return on ? "[ " + low + ", " + high + " ]" : "OFF";
     }
   }

   /**
    * Bitrate limits for resolution.
    */
   public class ResolutionBitrateLimits {
     /**
      * Maximum size of video frame, in pixels, the bitrate limits are intended for.
      */
     public final int frameSizePixels;

     /**
      * Recommended minimum bitrate to start encoding.
      */
     public final int minStartBitrateBps;

     /**
      * Recommended minimum bitrate.
      */
     public final int minBitrateBps;

     /**
      * Recommended maximum bitrate.
      */
     public final int maxBitrateBps;

     public ResolutionBitrateLimits(
         int frameSizePixels, int minStartBitrateBps, int minBitrateBps, int maxBitrateBps) {
       this.frameSizePixels = frameSizePixels;
       this.minStartBitrateBps = minStartBitrateBps;
       this.minBitrateBps = minBitrateBps;
       this.maxBitrateBps = maxBitrateBps;
     }

     @CalledByNative("ResolutionBitrateLimits")
     public int getFrameSizePixels() {
       return frameSizePixels;
     }

     @CalledByNative("ResolutionBitrateLimits")
     public int getMinStartBitrateBps() {
       return minStartBitrateBps;
     }

     @CalledByNative("ResolutionBitrateLimits")
     public int getMinBitrateBps() {
       return minBitrateBps;
     }

     @CalledByNative("ResolutionBitrateLimits")
     public int getMaxBitrateBps() {
       return maxBitrateBps;
     }
   }

   /** Rate control parameters. */
   public class RateControlParameters {
     /**
      * Adjusted target bitrate, per spatial/temporal layer. May be lower or higher than the target
      * depending on encoder behaviour.
      */
     public final BitrateAllocation bitrate;

     /**
      * Target framerate, in fps. A value <= 0.0 is invalid and should be interpreted as framerate
      * target not available. In this case the encoder should fall back to the max framerate
      * specified in `codec_settings` of the last InitEncode() call.
      */
     public final double framerateFps;

     @CalledByNative("RateControlParameters")
     public RateControlParameters(BitrateAllocation bitrate, double framerateFps) {
       this.bitrate = bitrate;
       this.framerateFps = framerateFps;
     }
   }

   /**
    * Metadata about the Encoder.
    */
   public class EncoderInfo {
     /**
      * The width and height of the incoming video frames should be divisible by
      * |requested_resolution_alignment|
      */
     public final int requestedResolutionAlignment;

     /**
      * Same as above but if true, each simulcast layer should also be divisible by
      * |requested_resolution_alignment|.
      */
     public final boolean applyAlignmentToAllSimulcastLayers;

     public EncoderInfo(
         int requestedResolutionAlignment, boolean applyAlignmentToAllSimulcastLayers) {
       this.requestedResolutionAlignment = requestedResolutionAlignment;
       this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers;
     }

     @CalledByNative("EncoderInfo")
     public int getRequestedResolutionAlignment() {
       return requestedResolutionAlignment;
     }

     @CalledByNative("EncoderInfo")
     public boolean getApplyAlignmentToAllSimulcastLayers() {
       return applyAlignmentToAllSimulcastLayers;
     }
   }

   public interface Callback {
     /**
      * Old encoders assume that the byte buffer held by `frame` is not accessed after the call to
      * this method returns. If the pipeline downstream needs to hold on to the buffer, it then has
      * to make its own copy. We want to move to a model where no copying is needed, and instead use
      * retain()/release() to signal to the encoder when it is safe to reuse the buffer.
      *
      * Over the transition, implementations of this class should use the maybeRetain() method if
      * they want to keep a reference to the buffer, and fall back to copying if that method returns
      * false.
      */
     void onEncodedFrame(EncodedImage frame, CodecSpecificInfo info);
   }

   /**
    * The encoder implementation backing this interface is either 1) a Java
    * encoder (e.g., an Android platform encoder), or alternatively 2) a native
    * encoder (e.g., a software encoder or a C++ encoder adapter).
    *
    * For case 1), createNativeVideoEncoder() should return zero.
    * In this case, we expect the native library to call the encoder through
    * JNI using the Java interface declared below.
    *
    * For case 2), createNativeVideoEncoder() should return a non-zero value.
    * In this case, we expect the native library to treat the returned value as
    * a raw pointer of type webrtc::VideoEncoder* (ownership is transferred to
    * the caller). The native library should then directly call the
    * webrtc::VideoEncoder interface without going through JNI. All calls to
    * the Java interface methods declared below should thus throw an
    * UnsupportedOperationException.
    */
   @CalledByNative
   default long createNativeVideoEncoder() {
     return 0;
   }

   /**
    * Returns true if the encoder is backed by hardware.
    */
   @CalledByNative
   default boolean isHardwareEncoder() {
     return true;
   }

   /**
    * Initializes the encoding process. Call before any calls to encode.
    */
   @CalledByNative VideoCodecStatus initEncode(Settings settings, Callback encodeCallback);

   /**
    * Releases the encoder. No more calls to encode will be made after this call.
    */
   @CalledByNative VideoCodecStatus release();

   /**
    * Requests the encoder to encode a frame.
    */
   @CalledByNative VideoCodecStatus encode(VideoFrame frame, EncodeInfo info);

   /** Sets the bitrate allocation and the target framerate for the encoder. */
   VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate);

   /** Sets the bitrate allocation and the target framerate for the encoder. */
   default @CalledByNative VideoCodecStatus setRates(RateControlParameters rcParameters) {
     // Round frame rate up to avoid overshoots.
     int framerateFps = (int) Math.ceil(rcParameters.framerateFps);
     return setRateAllocation(rcParameters.bitrate, framerateFps);
   }

   /** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */
   @CalledByNative ScalingSettings getScalingSettings();

   /** Returns the list of bitrate limits. */
   @CalledByNative
   default ResolutionBitrateLimits[] getResolutionBitrateLimits() {
     // TODO(ssilkin): Update downstream projects and remove default implementation.
     ResolutionBitrateLimits bitrate_limits[] = {};
     return bitrate_limits;
   }

   /**
    * Should return a descriptive name for the implementation. Gets called once and cached. May be
    * called from arbitrary thread.
    */
   @CalledByNative String getImplementationName();

   @CalledByNative
   default EncoderInfo getEncoderInfo() {
     return new EncoderInfo(
         /* requestedResolutionAlignment= */ 1, /* applyAlignmentToAllSimulcastLayers= */ false);
   }
 }
	/*
	* Copyright 2017 The WebRTC project authors. All Rights Reserved.
	*
	* Use of this source code is governed by a BSD-style license
	* that can be found in the LICENSE file in the root of the source
	* tree. An additional intellectual property rights grant can be found
	* in the file PATENTS. All contributing project authors may
	* be found in the AUTHORS file in the root of the source tree.
	*/

	package org.webrtc;

	import androidx.annotation.Nullable;
	import org.webrtc.EncodedImage;

	/**
	* Interface for a video encoder that can be used with WebRTC. All calls will be made on the
	* encoding thread. The encoder may be constructed on a different thread and changing thread after
	* calling release is allowed.
	*/
	public interface VideoEncoder {
	/** Settings passed to the encoder by WebRTC. */
	public class Settings {
	public final int numberOfCores;
	public final int width;
	public final int height;
	public final int startBitrate; // Kilobits per second.
	public final int maxFramerate;
	public final int numberOfSimulcastStreams;
	public final boolean automaticResizeOn;
	public final Capabilities capabilities;

	// TODO(bugs.webrtc.org/10720): Remove.
	@Deprecated
	public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate,
	int numberOfSimulcastStreams, boolean automaticResizeOn) {
	this(numberOfCores, width, height, startBitrate, maxFramerate, numberOfSimulcastStreams,
	automaticResizeOn, new VideoEncoder.Capabilities(false /* lossNotification */));
	}

	@CalledByNative("Settings")
	public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate,
	int numberOfSimulcastStreams, boolean automaticResizeOn, Capabilities capabilities) {
	this.numberOfCores = numberOfCores;
	this.width = width;
	this.height = height;
	this.startBitrate = startBitrate;
	this.maxFramerate = maxFramerate;
	this.numberOfSimulcastStreams = numberOfSimulcastStreams;
	this.automaticResizeOn = automaticResizeOn;
	this.capabilities = capabilities;
	}
	}

	/** Capabilities (loss notification, etc.) passed to the encoder by WebRTC. */
	public class Capabilities {
	/**
	* The remote side has support for the loss notification RTCP feedback message format, and will
	* be sending these feedback messages if necessary.
	*/
	public final boolean lossNotification;

	@CalledByNative("Capabilities")
	public Capabilities(boolean lossNotification) {
	this.lossNotification = lossNotification;
	}
	}

	/** Additional info for encoding. */
	public class EncodeInfo {
	public final EncodedImage.FrameType[] frameTypes;

	@CalledByNative("EncodeInfo")
	public EncodeInfo(EncodedImage.FrameType[] frameTypes) {
	this.frameTypes = frameTypes;
	}
	}

	// TODO(sakal): Add values to these classes as necessary.
	/** Codec specific information about the encoded frame. */
	public class CodecSpecificInfo {}

	public class CodecSpecificInfoVP8 extends CodecSpecificInfo {}

	public class CodecSpecificInfoVP9 extends CodecSpecificInfo {}

	public class CodecSpecificInfoH264 extends CodecSpecificInfo {}

	public class CodecSpecificInfoAV1 extends CodecSpecificInfo {}

	/**
	* Represents bitrate allocated for an encoder to produce frames. Bitrate can be divided between
	* spatial and temporal layers.
	*/
	public class BitrateAllocation {
	// First index is the spatial layer and second the temporal layer.
	public final int[][] bitratesBbs;

	/**
	* Initializes the allocation with a two dimensional array of bitrates. The first index of the
	* array is the spatial layer and the second index in the temporal layer.
	*/
	@CalledByNative("BitrateAllocation")
	public BitrateAllocation(int[][] bitratesBbs) {
	this.bitratesBbs = bitratesBbs;
	}

	/**
	* Gets the total bitrate allocated for all layers.
	*/
	public int getSum() {
	int sum = 0;
	for (int[] spatialLayer : bitratesBbs) {
	for (int bitrate : spatialLayer) {
	sum += bitrate;
	}
	}
	return sum;
	}
	}

	/** Settings for WebRTC quality based scaling. */
	public class ScalingSettings {
	public final boolean on;
	@Nullable public final Integer low;
	@Nullable public final Integer high;

	/**
	* Settings to disable quality based scaling.
	*/
	public static final ScalingSettings OFF = new ScalingSettings();

	/**
	* Creates settings to enable quality based scaling.
	*
	* @param low Average QP at which to scale up the resolution.
	* @param high Average QP at which to scale down the resolution.
	*/
	public ScalingSettings(int low, int high) {
	this.on = true;
	this.low = low;
	this.high = high;
	}

	private ScalingSettings() {
	this.on = false;
	this.low = null;
	this.high = null;
	}

	// TODO(bugs.webrtc.org/8830): Below constructors are deprecated.
	// Default thresholds are going away, so thresholds have to be set
	// when scaling is on.
	/**
	* Creates quality based scaling setting.
	*
	* @param on True if quality scaling is turned on.
	*/
	@Deprecated
	public ScalingSettings(boolean on) {
	this.on = on;
	this.low = null;
	this.high = null;
	}

	/**
	* Creates quality based scaling settings with custom thresholds.
	*
	* @param on True if quality scaling is turned on.
	* @param low Average QP at which to scale up the resolution.
	* @param high Average QP at which to scale down the resolution.
	*/
	@Deprecated
	public ScalingSettings(boolean on, int low, int high) {
	this.on = on;
	this.low = low;
	this.high = high;
	}

	@Override
	public String toString() {
	return on ? "[ " + low + ", " + high + " ]" : "OFF";
	}
	}

	/**
	* Bitrate limits for resolution.
	*/
	public class ResolutionBitrateLimits {
	/**
	* Maximum size of video frame, in pixels, the bitrate limits are intended for.
	*/
	public final int frameSizePixels;

	/**
	* Recommended minimum bitrate to start encoding.
	*/
	public final int minStartBitrateBps;

	/**
	* Recommended minimum bitrate.
	*/
	public final int minBitrateBps;

	/**
	* Recommended maximum bitrate.
	*/
	public final int maxBitrateBps;

	public ResolutionBitrateLimits(
	int frameSizePixels, int minStartBitrateBps, int minBitrateBps, int maxBitrateBps) {
	this.frameSizePixels = frameSizePixels;
	this.minStartBitrateBps = minStartBitrateBps;
	this.minBitrateBps = minBitrateBps;
	this.maxBitrateBps = maxBitrateBps;
	}

	@CalledByNative("ResolutionBitrateLimits")
	public int getFrameSizePixels() {
	return frameSizePixels;
	}

	@CalledByNative("ResolutionBitrateLimits")
	public int getMinStartBitrateBps() {
	return minStartBitrateBps;
	}

	@CalledByNative("ResolutionBitrateLimits")
	public int getMinBitrateBps() {
	return minBitrateBps;
	}

	@CalledByNative("ResolutionBitrateLimits")
	public int getMaxBitrateBps() {
	return maxBitrateBps;
	}
	}

	/** Rate control parameters. */
	public class RateControlParameters {
	/**
	* Adjusted target bitrate, per spatial/temporal layer. May be lower or higher than the target
	* depending on encoder behaviour.
	*/
	public final BitrateAllocation bitrate;

	/**
	* Target framerate, in fps. A value <= 0.0 is invalid and should be interpreted as framerate
	* target not available. In this case the encoder should fall back to the max framerate
	* specified in `codec_settings` of the last InitEncode() call.
	*/
	public final double framerateFps;

	@CalledByNative("RateControlParameters")
	public RateControlParameters(BitrateAllocation bitrate, double framerateFps) {
	this.bitrate = bitrate;
	this.framerateFps = framerateFps;
	}
	}

	/**
	* Metadata about the Encoder.
	*/
	public class EncoderInfo {
	/**
	* The width and height of the incoming video frames should be divisible by
	* \|requested_resolution_alignment\|
	*/
	public final int requestedResolutionAlignment;

	/**
	* Same as above but if true, each simulcast layer should also be divisible by
	* \|requested_resolution_alignment\|.
	*/
	public final boolean applyAlignmentToAllSimulcastLayers;

	public EncoderInfo(
	int requestedResolutionAlignment, boolean applyAlignmentToAllSimulcastLayers) {
	this.requestedResolutionAlignment = requestedResolutionAlignment;
	this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers;
	}

	@CalledByNative("EncoderInfo")
	public int getRequestedResolutionAlignment() {
	return requestedResolutionAlignment;
	}

	@CalledByNative("EncoderInfo")
	public boolean getApplyAlignmentToAllSimulcastLayers() {
	return applyAlignmentToAllSimulcastLayers;
	}
	}

	public interface Callback {
	/**
	* Old encoders assume that the byte buffer held by `frame` is not accessed after the call to
	* this method returns. If the pipeline downstream needs to hold on to the buffer, it then has
	* to make its own copy. We want to move to a model where no copying is needed, and instead use
	* retain()/release() to signal to the encoder when it is safe to reuse the buffer.
	*
	* Over the transition, implementations of this class should use the maybeRetain() method if
	* they want to keep a reference to the buffer, and fall back to copying if that method returns
	* false.
	*/
	void onEncodedFrame(EncodedImage frame, CodecSpecificInfo info);
	}

	/**
	* The encoder implementation backing this interface is either 1) a Java
	* encoder (e.g., an Android platform encoder), or alternatively 2) a native
	* encoder (e.g., a software encoder or a C++ encoder adapter).
	*
	* For case 1), createNativeVideoEncoder() should return zero.
	* In this case, we expect the native library to call the encoder through
	* JNI using the Java interface declared below.
	*
	* For case 2), createNativeVideoEncoder() should return a non-zero value.
	* In this case, we expect the native library to treat the returned value as
	* a raw pointer of type webrtc::VideoEncoder* (ownership is transferred to
	* the caller). The native library should then directly call the
	* webrtc::VideoEncoder interface without going through JNI. All calls to
	* the Java interface methods declared below should thus throw an
	* UnsupportedOperationException.
	*/
	@CalledByNative
	default long createNativeVideoEncoder() {
	return 0;
	}

	/**
	* Returns true if the encoder is backed by hardware.
	*/
	@CalledByNative
	default boolean isHardwareEncoder() {
	return true;
	}

	/**
	* Initializes the encoding process. Call before any calls to encode.
	*/
	@CalledByNative VideoCodecStatus initEncode(Settings settings, Callback encodeCallback);

	/**
	* Releases the encoder. No more calls to encode will be made after this call.
	*/
	@CalledByNative VideoCodecStatus release();

	/**
	* Requests the encoder to encode a frame.
	*/
	@CalledByNative VideoCodecStatus encode(VideoFrame frame, EncodeInfo info);

	/** Sets the bitrate allocation and the target framerate for the encoder. */
	VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate);

	/** Sets the bitrate allocation and the target framerate for the encoder. */
	default @CalledByNative VideoCodecStatus setRates(RateControlParameters rcParameters) {
	// Round frame rate up to avoid overshoots.
	int framerateFps = (int) Math.ceil(rcParameters.framerateFps);
	return setRateAllocation(rcParameters.bitrate, framerateFps);
	}

	/** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */
	@CalledByNative ScalingSettings getScalingSettings();

	/** Returns the list of bitrate limits. */
	@CalledByNative
	default ResolutionBitrateLimits[] getResolutionBitrateLimits() {
	// TODO(ssilkin): Update downstream projects and remove default implementation.
	ResolutionBitrateLimits bitrate_limits[] = {};
	return bitrate_limits;
	}

	/**
	* Should return a descriptive name for the implementation. Gets called once and cached. May be
	* called from arbitrary thread.
	*/
	@CalledByNative String getImplementationName();

	@CalledByNative
	default EncoderInfo getEncoderInfo() {
	return new EncoderInfo(
	/* requestedResolutionAlignment= / 1, / applyAlignmentToAllSimulcastLayers= */ false);
	}
	}