sdk/android/api/org/webrtc/RendererCommon.java

/*
 *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

package org.webrtc;

import android.graphics.Point;
import android.opengl.Matrix;
import android.view.View;

/**
 * Static helper functions for renderer implementations.
 */
public class RendererCommon {
  /** Interface for reporting rendering events. */
  public static interface RendererEvents {
    /**
     * Callback fired once first frame is rendered.
     */
    public void onFirstFrameRendered();

    /**
     * Callback fired when rendered frame resolution or rotation has changed.
     */
    public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation);
  }

  /**
   * Interface for rendering frames on an EGLSurface with specified viewport location. Rotation,
   * mirror, and cropping is specified using a 4x4 texture coordinate transform matrix. The frame
   * input can either be an OES texture, RGB texture, or YUV textures in I420 format. The function
   * release() must be called manually to free the resources held by this object.
   */
  public static interface GlDrawer {
    /**
     * Functions for drawing frames with different sources. The rendering surface target is
     * implied by the current EGL context of the calling thread and requires no explicit argument.
     * The coordinates specify the viewport location on the surface target.
     */
    void drawOes(int oesTextureId, float[] texMatrix, int frameWidth, int frameHeight,
        int viewportX, int viewportY, int viewportWidth, int viewportHeight);
    void drawRgb(int textureId, float[] texMatrix, int frameWidth, int frameHeight, int viewportX,
        int viewportY, int viewportWidth, int viewportHeight);
    void drawYuv(int[] yuvTextures, float[] texMatrix, int frameWidth, int frameHeight,
        int viewportX, int viewportY, int viewportWidth, int viewportHeight);

    /**
     * Release all GL resources. This needs to be done manually, otherwise resources may leak.
     */
    void release();
  }

  /**
   * Helper class for determining layout size based on layout requirements, scaling type, and video
   * aspect ratio.
   */
  public static class VideoLayoutMeasure {
    // The scaling type determines how the video will fill the allowed layout area in measure(). It
    // can be specified separately for the case when video has matched orientation with layout size
    // and when there is an orientation mismatch.
    private float visibleFractionMatchOrientation =
        convertScalingTypeToVisibleFraction(ScalingType.SCALE_ASPECT_BALANCED);
    private float visibleFractionMismatchOrientation =
        convertScalingTypeToVisibleFraction(ScalingType.SCALE_ASPECT_BALANCED);

    public void setScalingType(ScalingType scalingType) {
      setScalingType(/* scalingTypeMatchOrientation= */ scalingType,
          /* scalingTypeMismatchOrientation= */ scalingType);
    }

    public void setScalingType(
        ScalingType scalingTypeMatchOrientation, ScalingType scalingTypeMismatchOrientation) {
      this.visibleFractionMatchOrientation =
          convertScalingTypeToVisibleFraction(scalingTypeMatchOrientation);
      this.visibleFractionMismatchOrientation =
          convertScalingTypeToVisibleFraction(scalingTypeMismatchOrientation);
    }

    public void setVisibleFraction(
        float visibleFractionMatchOrientation, float visibleFractionMismatchOrientation) {
      this.visibleFractionMatchOrientation = visibleFractionMatchOrientation;
      this.visibleFractionMismatchOrientation = visibleFractionMismatchOrientation;
    }

    public Point measure(int widthSpec, int heightSpec, int frameWidth, int frameHeight) {
      // Calculate max allowed layout size.
      final int maxWidth = View.getDefaultSize(Integer.MAX_VALUE, widthSpec);
      final int maxHeight = View.getDefaultSize(Integer.MAX_VALUE, heightSpec);
      if (frameWidth == 0 || frameHeight == 0 || maxWidth == 0 || maxHeight == 0) {
        return new Point(maxWidth, maxHeight);
      }
      // Calculate desired display size based on scaling type, video aspect ratio,
      // and maximum layout size.
      final float frameAspect = frameWidth / (float) frameHeight;
      final float displayAspect = maxWidth / (float) maxHeight;
      final float visibleFraction = (frameAspect > 1.0f) == (displayAspect > 1.0f)
          ? visibleFractionMatchOrientation
          : visibleFractionMismatchOrientation;
      final Point layoutSize = getDisplaySize(visibleFraction, frameAspect, maxWidth, maxHeight);

      // If the measure specification is forcing a specific size - yield.
      if (View.MeasureSpec.getMode(widthSpec) == View.MeasureSpec.EXACTLY) {
        layoutSize.x = maxWidth;
      }
      if (View.MeasureSpec.getMode(heightSpec) == View.MeasureSpec.EXACTLY) {
        layoutSize.y = maxHeight;
      }
      return layoutSize;
    }
  }

  // Types of video scaling:
  // SCALE_ASPECT_FIT - video frame is scaled to fit the size of the view by
  //    maintaining the aspect ratio (black borders may be displayed).
  // SCALE_ASPECT_FILL - video frame is scaled to fill the size of the view by
  //    maintaining the aspect ratio. Some portion of the video frame may be
  //    clipped.
  // SCALE_ASPECT_BALANCED - Compromise between FIT and FILL. Video frame will fill as much as
  // possible of the view while maintaining aspect ratio, under the constraint that at least
  // `BALANCED_VISIBLE_FRACTION` of the frame content will be shown.
  public static enum ScalingType { SCALE_ASPECT_FIT, SCALE_ASPECT_FILL, SCALE_ASPECT_BALANCED }
  // The minimum fraction of the frame content that will be shown for `SCALE_ASPECT_BALANCED`.
  // This limits excessive cropping when adjusting display size.
  private static float BALANCED_VISIBLE_FRACTION = 0.5625f;

  /**
   * Returns layout transformation matrix that applies an optional mirror effect and compensates
   * for video vs display aspect ratio.
   */
  public static float[] getLayoutMatrix(
      boolean mirror, float videoAspectRatio, float displayAspectRatio) {
    float scaleX = 1;
    float scaleY = 1;
    // Scale X or Y dimension so that video and display size have same aspect ratio.
    if (displayAspectRatio > videoAspectRatio) {
      scaleY = videoAspectRatio / displayAspectRatio;
    } else {
      scaleX = displayAspectRatio / videoAspectRatio;
    }
    // Apply optional horizontal flip.
    if (mirror) {
      scaleX *= -1;
    }
    final float matrix[] = new float[16];
    Matrix.setIdentityM(matrix, 0);
    Matrix.scaleM(matrix, 0, scaleX, scaleY, 1);
    adjustOrigin(matrix);
    return matrix;
  }

  /** Converts a float[16] matrix array to android.graphics.Matrix. */
  public static android.graphics.Matrix convertMatrixToAndroidGraphicsMatrix(float[] matrix4x4) {
    // clang-format off
    float[] values = {
        matrix4x4[0 * 4 + 0], matrix4x4[1 * 4 + 0], matrix4x4[3 * 4 + 0],
        matrix4x4[0 * 4 + 1], matrix4x4[1 * 4 + 1], matrix4x4[3 * 4 + 1],
        matrix4x4[0 * 4 + 3], matrix4x4[1 * 4 + 3], matrix4x4[3 * 4 + 3],
    };
    // clang-format on

    android.graphics.Matrix matrix = new android.graphics.Matrix();
    matrix.setValues(values);
    return matrix;
  }

  /** Converts android.graphics.Matrix to a float[16] matrix array. */
  public static float[] convertMatrixFromAndroidGraphicsMatrix(android.graphics.Matrix matrix) {
    float[] values = new float[9];
    matrix.getValues(values);

    // The android.graphics.Matrix looks like this:
    // [x1 y1 w1]
    // [x2 y2 w2]
    // [x3 y3 w3]
    // We want to contruct a matrix that looks like this:
    // [x1 y1  0 w1]
    // [x2 y2  0 w2]
    // [ 0  0  1  0]
    // [x3 y3  0 w3]
    // Since it is stored in column-major order, it looks like this:
    // [x1 x2 0 x3
    //  y1 y2 0 y3
    //   0  0 1  0
    //  w1 w2 0 w3]
    // clang-format off
    float[] matrix4x4 = {
        values[0 * 3 + 0],  values[1 * 3 + 0], 0,  values[2 * 3 + 0],
        values[0 * 3 + 1],  values[1 * 3 + 1], 0,  values[2 * 3 + 1],
        0,                  0,                 1,  0,
        values[0 * 3 + 2],  values[1 * 3 + 2], 0,  values[2 * 3 + 2],
    };
    // clang-format on
    return matrix4x4;
  }

  /**
   * Calculate display size based on scaling type, video aspect ratio, and maximum display size.
   */
  public static Point getDisplaySize(
      ScalingType scalingType, float videoAspectRatio, int maxDisplayWidth, int maxDisplayHeight) {
    return getDisplaySize(convertScalingTypeToVisibleFraction(scalingType), videoAspectRatio,
        maxDisplayWidth, maxDisplayHeight);
  }

  /**
   * Move `matrix` transformation origin to (0.5, 0.5). This is the origin for texture coordinates
   * that are in the range 0 to 1.
   */
  private static void adjustOrigin(float[] matrix) {
    // Note that OpenGL is using column-major order.
    // Pre translate with -0.5 to move coordinates to range [-0.5, 0.5].
    matrix[12] -= 0.5f * (matrix[0] + matrix[4]);
    matrix[13] -= 0.5f * (matrix[1] + matrix[5]);
    // Post translate with 0.5 to move coordinates to range [0, 1].
    matrix[12] += 0.5f;
    matrix[13] += 0.5f;
  }

  /**
   * Each scaling type has a one-to-one correspondence to a numeric minimum fraction of the video
   * that must remain visible.
   */
  private static float convertScalingTypeToVisibleFraction(ScalingType scalingType) {
    switch (scalingType) {
      case SCALE_ASPECT_FIT:
        return 1.0f;
      case SCALE_ASPECT_FILL:
        return 0.0f;
      case SCALE_ASPECT_BALANCED:
        return BALANCED_VISIBLE_FRACTION;
      default:
        throw new IllegalArgumentException();
    }
  }

  /**
   * Calculate display size based on minimum fraction of the video that must remain visible,
   * video aspect ratio, and maximum display size.
   */
  public static Point getDisplaySize(
      float minVisibleFraction, float videoAspectRatio, int maxDisplayWidth, int maxDisplayHeight) {
    // If there is no constraint on the amount of cropping, fill the allowed display area.
    if (minVisibleFraction == 0 || videoAspectRatio == 0) {
      return new Point(maxDisplayWidth, maxDisplayHeight);
    }
    // Each dimension is constrained on max display size and how much we are allowed to crop.
    final int width = Math.min(
        maxDisplayWidth, Math.round(maxDisplayHeight / minVisibleFraction * videoAspectRatio));
    final int height = Math.min(
        maxDisplayHeight, Math.round(maxDisplayWidth / minVisibleFraction / videoAspectRatio));
    return new Point(width, height);
  }
}