sdk/android/api/org/webrtc/VideoFrameDrawer.java - src.git - Git at Google

 /*
  *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */

 package org.webrtc;

 import android.graphics.Matrix;
 import android.graphics.Point;
 import android.opengl.GLES20;
 import androidx.annotation.Nullable;
 import java.nio.ByteBuffer;

 /**
  * Helper class to draw VideoFrames. Calls either drawer.drawOes, drawer.drawRgb, or
  * drawer.drawYuv depending on the type of the buffer. The frame will be rendered with rotation
  * taken into account. You can supply an additional render matrix for custom transformations.
  */
 public class VideoFrameDrawer {
   public static final String TAG = "VideoFrameDrawer";
   /**
    * Draws a VideoFrame.TextureBuffer. Calls either drawer.drawOes or drawer.drawRgb
    * depending on the type of the buffer. You can supply an additional render matrix. This is
    * used multiplied together with the transformation matrix of the frame. (M = renderMatrix *
    * transformationMatrix)
    */
   public static void drawTexture(RendererCommon.GlDrawer drawer, VideoFrame.TextureBuffer buffer,
       Matrix renderMatrix, int frameWidth, int frameHeight, int viewportX, int viewportY,
       int viewportWidth, int viewportHeight) {
     Matrix finalMatrix = new Matrix(buffer.getTransformMatrix());
     finalMatrix.preConcat(renderMatrix);
     float[] finalGlMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(finalMatrix);
     switch (buffer.getType()) {
       case OES:
         drawer.drawOes(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX,
             viewportY, viewportWidth, viewportHeight);
         break;
       case RGB:
         drawer.drawRgb(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX,
             viewportY, viewportWidth, viewportHeight);
         break;
       default:
         throw new RuntimeException("Unknown texture type.");
     }
   }

   /**
    * Helper class for uploading YUV bytebuffer frames to textures that handles stride > width. This
    * class keeps an internal ByteBuffer to avoid unnecessary allocations for intermediate copies.
    */
   private static class YuvUploader {
     // Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width.
     // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader
     // that handles stride and compare performance with intermediate copy.
     @Nullable private ByteBuffer copyBuffer;
     @Nullable private int[] yuvTextures;

     /**
      * Upload `planes` into OpenGL textures, taking stride into consideration.
      *
      * @return Array of three texture indices corresponding to Y-, U-, and V-plane respectively.
      */
     @Nullable
     public int[] uploadYuvData(int width, int height, int[] strides, ByteBuffer[] planes) {
       final int[] planeWidths = new int[] {width, width / 2, width / 2};
       final int[] planeHeights = new int[] {height, height / 2, height / 2};
       // Make a first pass to see if we need a temporary copy buffer.
       int copyCapacityNeeded = 0;
       for (int i = 0; i < 3; ++i) {
         if (strides[i] > planeWidths[i]) {
           copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidths[i] * planeHeights[i]);
         }
       }
       // Allocate copy buffer if necessary.
       if (copyCapacityNeeded > 0
           && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) {
         copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded);
       }
       // Make sure YUV textures are allocated.
       if (yuvTextures == null) {
         yuvTextures = new int[3];
         for (int i = 0; i < 3; i++) {
           yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
         }
       }
       // Upload each plane.
       for (int i = 0; i < 3; ++i) {
         GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
         GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
         // GLES only accepts packed data, i.e. stride == planeWidth.
         final ByteBuffer packedByteBuffer;
         if (strides[i] == planeWidths[i]) {
           // Input is packed already.
           packedByteBuffer = planes[i];
         } else {
           YuvHelper.copyPlane(
               planes[i], strides[i], copyBuffer, planeWidths[i], planeWidths[i], planeHeights[i]);
           packedByteBuffer = copyBuffer;
         }
         GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidths[i],
             planeHeights[i], 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer);
       }
       return yuvTextures;
     }

     @Nullable
     public int[] uploadFromBuffer(VideoFrame.I420Buffer buffer) {
       int[] strides = {buffer.getStrideY(), buffer.getStrideU(), buffer.getStrideV()};
       ByteBuffer[] planes = {buffer.getDataY(), buffer.getDataU(), buffer.getDataV()};
       return uploadYuvData(buffer.getWidth(), buffer.getHeight(), strides, planes);
     }

     @Nullable
     public int[] getYuvTextures() {
       return yuvTextures;
     }

     /**
      * Releases cached resources. Uploader can still be used and the resources will be reallocated
      * on first use.
      */
     public void release() {
       copyBuffer = null;
       if (yuvTextures != null) {
         GLES20.glDeleteTextures(3, yuvTextures, 0);
         yuvTextures = null;
       }
     }
   }

   private static int distance(float x0, float y0, float x1, float y1) {
     return (int) Math.round(Math.hypot(x1 - x0, y1 - y0));
   }

   // These points are used to calculate the size of the part of the frame we are rendering.
   final static float[] srcPoints =
       new float[] {0f /* x0 */, 0f /* y0 */, 1f /* x1 */, 0f /* y1 */, 0f /* x2 */, 1f /* y2 */};
   private final float[] dstPoints = new float[6];
   private final Point renderSize = new Point();
   private int renderWidth;
   private int renderHeight;

   // Calculate the frame size after `renderMatrix` is applied. Stores the output in member variables
   // `renderWidth` and `renderHeight` to avoid allocations since this function is called for every
   // frame.
   private void calculateTransformedRenderSize(
       int frameWidth, int frameHeight, @Nullable Matrix renderMatrix) {
     if (renderMatrix == null) {
       renderWidth = frameWidth;
       renderHeight = frameHeight;
       return;
     }
     // Transform the texture coordinates (in the range [0, 1]) according to `renderMatrix`.
     renderMatrix.mapPoints(dstPoints, srcPoints);

     // Multiply with the width and height to get the positions in terms of pixels.
     for (int i = 0; i < 3; ++i) {
       dstPoints[i * 2 + 0] *= frameWidth;
       dstPoints[i * 2 + 1] *= frameHeight;
     }

     // Get the length of the sides of the transformed rectangle in terms of pixels.
     renderWidth = distance(dstPoints[0], dstPoints[1], dstPoints[2], dstPoints[3]);
     renderHeight = distance(dstPoints[0], dstPoints[1], dstPoints[4], dstPoints[5]);
   }

   private final YuvUploader yuvUploader = new YuvUploader();
   // This variable will only be used for checking reference equality and is used for caching I420
   // textures.
   @Nullable private VideoFrame lastI420Frame;
   private final Matrix renderMatrix = new Matrix();

   public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer) {
     drawFrame(frame, drawer, null /* additionalRenderMatrix */);
   }

   public void drawFrame(
       VideoFrame frame, RendererCommon.GlDrawer drawer, Matrix additionalRenderMatrix) {
     drawFrame(frame, drawer, additionalRenderMatrix, 0 /* viewportX */, 0 /* viewportY */,
         frame.getRotatedWidth(), frame.getRotatedHeight());
   }

   public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer,
       @Nullable Matrix additionalRenderMatrix, int viewportX, int viewportY, int viewportWidth,
       int viewportHeight) {
     final int width = frame.getRotatedWidth();
     final int height = frame.getRotatedHeight();
     calculateTransformedRenderSize(width, height, additionalRenderMatrix);
     if (renderWidth <= 0 || renderHeight <= 0) {
       Logging.w(TAG, "Illegal frame size: " + renderWidth + "x" + renderHeight);
       return;
     }

     final boolean isTextureFrame = frame.getBuffer() instanceof VideoFrame.TextureBuffer;
     renderMatrix.reset();
     renderMatrix.preTranslate(0.5f, 0.5f);
     if (!isTextureFrame) {
       renderMatrix.preScale(1f, -1f); // I420-frames are upside down
     }
     renderMatrix.preRotate(frame.getRotation());
     renderMatrix.preTranslate(-0.5f, -0.5f);
     if (additionalRenderMatrix != null) {
       renderMatrix.preConcat(additionalRenderMatrix);
     }

     if (isTextureFrame) {
       lastI420Frame = null;
       drawTexture(drawer, (VideoFrame.TextureBuffer) frame.getBuffer(), renderMatrix, renderWidth,
           renderHeight, viewportX, viewportY, viewportWidth, viewportHeight);
     } else {
       // Only upload the I420 data to textures once per frame, if we are called multiple times
       // with the same frame.
       if (frame != lastI420Frame) {
         lastI420Frame = frame;
         final VideoFrame.I420Buffer i420Buffer = frame.getBuffer().toI420();
         yuvUploader.uploadFromBuffer(i420Buffer);
         i420Buffer.release();
       }

       drawer.drawYuv(yuvUploader.getYuvTextures(),
           RendererCommon.convertMatrixFromAndroidGraphicsMatrix(renderMatrix), renderWidth,
           renderHeight, viewportX, viewportY, viewportWidth, viewportHeight);
     }
   }

   public VideoFrame.Buffer prepareBufferForViewportSize(
       VideoFrame.Buffer buffer, int width, int height) {
     buffer.retain();
     return buffer;
   }

   public void release() {
     yuvUploader.release();
     lastI420Frame = null;
   }
 }
	/*
	* Copyright 2017 The WebRTC project authors. All Rights Reserved.
	*
	* Use of this source code is governed by a BSD-style license
	* that can be found in the LICENSE file in the root of the source
	* tree. An additional intellectual property rights grant can be found
	* in the file PATENTS. All contributing project authors may
	* be found in the AUTHORS file in the root of the source tree.
	*/

	package org.webrtc;

	import android.graphics.Matrix;
	import android.graphics.Point;
	import android.opengl.GLES20;
	import androidx.annotation.Nullable;
	import java.nio.ByteBuffer;

	/**
	* Helper class to draw VideoFrames. Calls either drawer.drawOes, drawer.drawRgb, or
	* drawer.drawYuv depending on the type of the buffer. The frame will be rendered with rotation
	* taken into account. You can supply an additional render matrix for custom transformations.
	*/
	public class VideoFrameDrawer {
	public static final String TAG = "VideoFrameDrawer";
	/**
	* Draws a VideoFrame.TextureBuffer. Calls either drawer.drawOes or drawer.drawRgb
	* depending on the type of the buffer. You can supply an additional render matrix. This is
	* used multiplied together with the transformation matrix of the frame. (M = renderMatrix *
	* transformationMatrix)
	*/
	public static void drawTexture(RendererCommon.GlDrawer drawer, VideoFrame.TextureBuffer buffer,
	Matrix renderMatrix, int frameWidth, int frameHeight, int viewportX, int viewportY,
	int viewportWidth, int viewportHeight) {
	Matrix finalMatrix = new Matrix(buffer.getTransformMatrix());
	finalMatrix.preConcat(renderMatrix);
	float[] finalGlMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(finalMatrix);
	switch (buffer.getType()) {
	case OES:
	drawer.drawOes(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX,
	viewportY, viewportWidth, viewportHeight);
	break;
	case RGB:
	drawer.drawRgb(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX,
	viewportY, viewportWidth, viewportHeight);
	break;
	default:
	throw new RuntimeException("Unknown texture type.");
	}
	}

	/**
	* Helper class for uploading YUV bytebuffer frames to textures that handles stride > width. This
	* class keeps an internal ByteBuffer to avoid unnecessary allocations for intermediate copies.
	*/
	private static class YuvUploader {
	// Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width.
	// TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader
	// that handles stride and compare performance with intermediate copy.
	@Nullable private ByteBuffer copyBuffer;
	@Nullable private int[] yuvTextures;

	/**
	* Upload `planes` into OpenGL textures, taking stride into consideration.
	*
	* @return Array of three texture indices corresponding to Y-, U-, and V-plane respectively.
	*/
	@Nullable
	public int[] uploadYuvData(int width, int height, int[] strides, ByteBuffer[] planes) {
	final int[] planeWidths = new int[] {width, width / 2, width / 2};
	final int[] planeHeights = new int[] {height, height / 2, height / 2};
	// Make a first pass to see if we need a temporary copy buffer.
	int copyCapacityNeeded = 0;
	for (int i = 0; i < 3; ++i) {
	if (strides[i] > planeWidths[i]) {
	copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidths[i] * planeHeights[i]);
	}
	}
	// Allocate copy buffer if necessary.
	if (copyCapacityNeeded > 0
	&& (copyBuffer == null \|\| copyBuffer.capacity() < copyCapacityNeeded)) {
	copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded);
	}
	// Make sure YUV textures are allocated.
	if (yuvTextures == null) {
	yuvTextures = new int[3];
	for (int i = 0; i < 3; i++) {
	yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
	}
	}
	// Upload each plane.
	for (int i = 0; i < 3; ++i) {
	GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
	GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
	// GLES only accepts packed data, i.e. stride == planeWidth.
	final ByteBuffer packedByteBuffer;
	if (strides[i] == planeWidths[i]) {
	// Input is packed already.
	packedByteBuffer = planes[i];
	} else {
	YuvHelper.copyPlane(
	planes[i], strides[i], copyBuffer, planeWidths[i], planeWidths[i], planeHeights[i]);
	packedByteBuffer = copyBuffer;
	}
	GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidths[i],
	planeHeights[i], 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer);
	}
	return yuvTextures;
	}

	@Nullable
	public int[] uploadFromBuffer(VideoFrame.I420Buffer buffer) {
	int[] strides = {buffer.getStrideY(), buffer.getStrideU(), buffer.getStrideV()};
	ByteBuffer[] planes = {buffer.getDataY(), buffer.getDataU(), buffer.getDataV()};
	return uploadYuvData(buffer.getWidth(), buffer.getHeight(), strides, planes);
	}

	@Nullable
	public int[] getYuvTextures() {
	return yuvTextures;
	}

	/**
	* Releases cached resources. Uploader can still be used and the resources will be reallocated
	* on first use.
	*/
	public void release() {
	copyBuffer = null;
	if (yuvTextures != null) {
	GLES20.glDeleteTextures(3, yuvTextures, 0);
	yuvTextures = null;
	}
	}
	}

	private static int distance(float x0, float y0, float x1, float y1) {
	return (int) Math.round(Math.hypot(x1 - x0, y1 - y0));
	}

	// These points are used to calculate the size of the part of the frame we are rendering.
	final static float[] srcPoints =
	new float[] {0f /* x0 /, 0f / y0 /, 1f / x1 /, 0f / y1 /, 0f / x2 /, 1f / y2 */};
	private final float[] dstPoints = new float[6];
	private final Point renderSize = new Point();
	private int renderWidth;
	private int renderHeight;

	// Calculate the frame size after `renderMatrix` is applied. Stores the output in member variables
	// `renderWidth` and `renderHeight` to avoid allocations since this function is called for every
	// frame.
	private void calculateTransformedRenderSize(
	int frameWidth, int frameHeight, @Nullable Matrix renderMatrix) {
	if (renderMatrix == null) {
	renderWidth = frameWidth;
	renderHeight = frameHeight;
	return;
	}
	// Transform the texture coordinates (in the range [0, 1]) according to `renderMatrix`.
	renderMatrix.mapPoints(dstPoints, srcPoints);

	// Multiply with the width and height to get the positions in terms of pixels.
	for (int i = 0; i < 3; ++i) {
	dstPoints[i * 2 + 0] *= frameWidth;
	dstPoints[i * 2 + 1] *= frameHeight;
	}

	// Get the length of the sides of the transformed rectangle in terms of pixels.
	renderWidth = distance(dstPoints[0], dstPoints[1], dstPoints[2], dstPoints[3]);
	renderHeight = distance(dstPoints[0], dstPoints[1], dstPoints[4], dstPoints[5]);
	}

	private final YuvUploader yuvUploader = new YuvUploader();
	// This variable will only be used for checking reference equality and is used for caching I420
	// textures.
	@Nullable private VideoFrame lastI420Frame;
	private final Matrix renderMatrix = new Matrix();

	public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer) {
	drawFrame(frame, drawer, null /* additionalRenderMatrix */);
	}

	public void drawFrame(
	VideoFrame frame, RendererCommon.GlDrawer drawer, Matrix additionalRenderMatrix) {
	drawFrame(frame, drawer, additionalRenderMatrix, 0 /* viewportX /, 0 / viewportY */,
	frame.getRotatedWidth(), frame.getRotatedHeight());
	}

	public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer,
	@Nullable Matrix additionalRenderMatrix, int viewportX, int viewportY, int viewportWidth,
	int viewportHeight) {
	final int width = frame.getRotatedWidth();
	final int height = frame.getRotatedHeight();
	calculateTransformedRenderSize(width, height, additionalRenderMatrix);
	if (renderWidth <= 0 \|\| renderHeight <= 0) {
	Logging.w(TAG, "Illegal frame size: " + renderWidth + "x" + renderHeight);
	return;
	}

	final boolean isTextureFrame = frame.getBuffer() instanceof VideoFrame.TextureBuffer;
	renderMatrix.reset();
	renderMatrix.preTranslate(0.5f, 0.5f);
	if (!isTextureFrame) {
	renderMatrix.preScale(1f, -1f); // I420-frames are upside down
	}
	renderMatrix.preRotate(frame.getRotation());
	renderMatrix.preTranslate(-0.5f, -0.5f);
	if (additionalRenderMatrix != null) {
	renderMatrix.preConcat(additionalRenderMatrix);
	}

	if (isTextureFrame) {
	lastI420Frame = null;
	drawTexture(drawer, (VideoFrame.TextureBuffer) frame.getBuffer(), renderMatrix, renderWidth,
	renderHeight, viewportX, viewportY, viewportWidth, viewportHeight);
	} else {
	// Only upload the I420 data to textures once per frame, if we are called multiple times
	// with the same frame.
	if (frame != lastI420Frame) {
	lastI420Frame = frame;
	final VideoFrame.I420Buffer i420Buffer = frame.getBuffer().toI420();
	yuvUploader.uploadFromBuffer(i420Buffer);
	i420Buffer.release();
	}

	drawer.drawYuv(yuvUploader.getYuvTextures(),
	RendererCommon.convertMatrixFromAndroidGraphicsMatrix(renderMatrix), renderWidth,
	renderHeight, viewportX, viewportY, viewportWidth, viewportHeight);
	}
	}

	public VideoFrame.Buffer prepareBufferForViewportSize(
	VideoFrame.Buffer buffer, int width, int height) {
	buffer.retain();
	return buffer;
	}

	public void release() {
	yuvUploader.release();
	lastI420Frame = null;
	}
	}