VideoRendererGui: Move to async rendering and remove no longer needed code
BUG=webrtc:4742, webrtc:4909
R=glaznev@webrtc.org
Review URL: https://codereview.webrtc.org/1321853003 .
Cr-Commit-Position: refs/heads/master@{#9847}
diff --git a/talk/app/webrtc/androidtests/src/org/webrtc/GlRectDrawerTest.java b/talk/app/webrtc/androidtests/src/org/webrtc/GlRectDrawerTest.java
index 08b82af..71bd5e3 100644
--- a/talk/app/webrtc/androidtests/src/org/webrtc/GlRectDrawerTest.java
+++ b/talk/app/webrtc/androidtests/src/org/webrtc/GlRectDrawerTest.java
@@ -86,7 +86,7 @@
final GlRectDrawer drawer = new GlRectDrawer();
final float[] texMatrix = new float[16];
Matrix.setIdentityM(texMatrix, 0);
- drawer.drawYuv(WIDTH, HEIGHT, yuvTextures, texMatrix);
+ drawer.drawYuv(yuvTextures, texMatrix);
// Download the pixels in the pixel buffer as RGBA.
final ByteBuffer data = ByteBuffer.allocateDirect(WIDTH * HEIGHT * 4);
diff --git a/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java b/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java
index c14abb0..6b30321 100644
--- a/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java
+++ b/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java
@@ -120,6 +120,51 @@
private GlShader currentShader;
private float[] currentTexMatrix;
private int texMatrixLocation;
+ // Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width.
+ // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader that
+ // handles stride and compare performance with intermediate copy.
+ private ByteBuffer copyBuffer;
+
+ /**
+ * Upload |planes| into |outputYuvTextures|, taking stride into consideration. |outputYuvTextures|
+ * must have been generated in advance.
+ */
+ public void uploadYuvData(
+ int[] outputYuvTextures, int width, int height, int[] strides, ByteBuffer[] planes) {
+ // Make a first pass to see if we need a temporary copy buffer.
+ int copyCapacityNeeded = 0;
+ for (int i = 0; i < 3; ++i) {
+ final int planeWidth = (i == 0) ? width : width / 2;
+ final int planeHeight = (i == 0) ? height : height / 2;
+ if (strides[i] > planeWidth) {
+ copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidth * planeHeight);
+ }
+ }
+ // Allocate copy buffer if necessary.
+ if (copyCapacityNeeded > 0
+ && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) {
+ copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded);
+ }
+ // Upload each plane.
+ for (int i = 0; i < 3; ++i) {
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, outputYuvTextures[i]);
+ final int planeWidth = (i == 0) ? width : width / 2;
+ final int planeHeight = (i == 0) ? height : height / 2;
+ // GLES only accepts packed data, i.e. stride == planeWidth.
+ final ByteBuffer packedByteBuffer;
+ if (strides[i] == planeWidth) {
+ // Input is packed already.
+ packedByteBuffer = planes[i];
+ } else {
+ VideoRenderer.nativeCopyPlane(
+ planes[i], planeWidth, planeHeight, strides[i], copyBuffer, planeWidth);
+ packedByteBuffer = copyBuffer;
+ }
+ GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidth, planeHeight, 0,
+ GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer);
+ }
+ }
/**
* Draw an OES texture frame with specified texture transformation matrix. Required resources are
@@ -150,7 +195,7 @@
* Draw a YUV frame with specified texture transformation matrix. Required resources are
* allocated at the first call to this function.
*/
- public void drawYuv(int width, int height, int[] yuvTextures, float[] texMatrix) {
+ public void drawYuv(int[] yuvTextures, float[] texMatrix) {
prepareShader(YUV_FRAGMENT_SHADER_STRING);
// Bind the textures.
for (int i = 0; i < 3; ++i) {
@@ -219,5 +264,6 @@
shader.release();
}
shaders.clear();
+ copyBuffer = null;
}
}
diff --git a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
index 23da47b..8568007 100644
--- a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
+++ b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
@@ -29,7 +29,6 @@
import java.util.ArrayList;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
@@ -102,14 +101,11 @@
private int[] yuvTextures = { 0, 0, 0 };
private int oesTexture = 0;
- // Render frame queue - accessed by two threads. renderFrame() call does
- // an offer (writing I420Frame to render) and early-returns (recording
- // a dropped frame) if that queue is full. draw() call does a peek(),
- // copies frame to texture and then removes it from a queue using poll().
- private final LinkedBlockingQueue<I420Frame> frameToRenderQueue;
- // Local copy of incoming video frame. Synchronized on |frameToRenderQueue|.
- private I420Frame yuvFrameToRender;
- private I420Frame textureFrameToRender;
+ // Pending frame to render. Serves as a queue with size 1. |pendingFrame| is accessed by two
+ // threads - frames are received in renderFrame() and consumed in draw(). Frames are dropped in
+ // renderFrame() if the previous frame has not been rendered yet.
+ private I420Frame pendingFrame;
+ private final Object pendingFrameLock = new Object();
// Type of video frame used for recent frame rendering.
private static enum RendererType { RENDERER_YUV, RENDERER_TEXTURE };
private RendererType rendererType;
@@ -129,7 +125,7 @@
private long startTimeNs = -1;
// Time in ns spent in draw() function.
private long drawTimeNs;
- // Time in ns spent in renderFrame() function - including copying frame
+ // Time in ns spent in draw() copying resources from |pendingFrame| - including uploading frame
// data to rendering planes.
private long copyTimeNs;
// The allowed view area in percentage of screen size.
@@ -163,7 +159,6 @@
this.id = id;
this.scalingType = scalingType;
this.mirror = mirror;
- frameToRenderQueue = new LinkedBlockingQueue<I420Frame>(1);
layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
updateTextureProperties = false;
rotationDegree = 0;
@@ -171,10 +166,11 @@
private synchronized void release() {
surface = null;
- synchronized (frameToRenderQueue) {
- frameToRenderQueue.clear();
- yuvFrameToRender = null;
- textureFrameToRender = null;
+ synchronized (pendingFrameLock) {
+ if (pendingFrame != null) {
+ VideoRenderer.renderFrameDone(pendingFrame);
+ pendingFrame = null;
+ }
}
}
@@ -231,52 +227,47 @@
GLES20.glViewport(displayLayout.left, screenHeight - displayLayout.bottom,
displayLayout.width(), displayLayout.height());
- I420Frame frameFromQueue;
- synchronized (frameToRenderQueue) {
+ final boolean isNewFrame;
+ synchronized (pendingFrameLock) {
// Check if texture vertices/coordinates adjustment is required when
// screen orientation changes or video frame size changes.
checkAdjustTextureCoords();
- frameFromQueue = frameToRenderQueue.peek();
- if (frameFromQueue != null && startTimeNs == -1) {
+ isNewFrame = (pendingFrame != null);
+ if (isNewFrame && startTimeNs == -1) {
startTimeNs = now;
}
- if (frameFromQueue != null) {
- if (frameFromQueue.yuvFrame) {
- // YUV textures rendering. Upload YUV data as textures.
- for (int i = 0; i < 3; ++i) {
- GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
- GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
- int w = (i == 0) ? frameFromQueue.width : frameFromQueue.width / 2;
- int h = (i == 0) ? frameFromQueue.height : frameFromQueue.height / 2;
- GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE,
- w, h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
- frameFromQueue.yuvPlanes[i]);
- }
+ if (isNewFrame) {
+ if (pendingFrame.yuvFrame) {
+ rendererType = RendererType.RENDERER_YUV;
+ drawer.uploadYuvData(yuvTextures, pendingFrame.width, pendingFrame.height,
+ pendingFrame.yuvStrides, pendingFrame.yuvPlanes);
} else {
+ rendererType = RendererType.RENDERER_TEXTURE;
// External texture rendering. Copy texture id and update texture image to latest.
// TODO(magjed): We should not make an unmanaged copy of texture id. Also, this is not
// the best place to call updateTexImage.
- oesTexture = frameFromQueue.textureId;
- if (frameFromQueue.textureObject instanceof SurfaceTexture) {
+ oesTexture = pendingFrame.textureId;
+ if (pendingFrame.textureObject instanceof SurfaceTexture) {
SurfaceTexture surfaceTexture =
- (SurfaceTexture) frameFromQueue.textureObject;
+ (SurfaceTexture) pendingFrame.textureObject;
surfaceTexture.updateTexImage();
}
}
-
- frameToRenderQueue.poll();
+ copyTimeNs += (System.nanoTime() - now);
+ VideoRenderer.renderFrameDone(pendingFrame);
+ pendingFrame = null;
}
}
if (rendererType == RendererType.RENDERER_YUV) {
- drawer.drawYuv(videoWidth, videoHeight, yuvTextures, texMatrix);
+ drawer.drawYuv(yuvTextures, texMatrix);
} else {
drawer.drawOes(oesTexture, texMatrix);
}
- if (frameFromQueue != null) {
+ if (isNewFrame) {
framesRendered++;
drawTimeNs += (System.nanoTime() - now);
if ((framesRendered % 300) == 0) {
@@ -342,25 +333,13 @@
rendererEvents.onFrameResolutionChanged(videoWidth, videoHeight, rotation);
}
- // Frame re-allocation need to be synchronized with copying
- // frame to textures in draw() function to avoid re-allocating
- // the frame while it is being copied.
- synchronized (frameToRenderQueue) {
+ synchronized (updateTextureLock) {
Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setSize: " +
videoWidth + " x " + videoHeight + " rotation " + rotation);
this.videoWidth = videoWidth;
this.videoHeight = videoHeight;
rotationDegree = rotation;
- int[] strides = { videoWidth, videoWidth / 2, videoWidth / 2 };
-
- // Clear rendering queue.
- frameToRenderQueue.poll();
- // Re-allocate / allocate the frame.
- yuvFrameToRender = new I420Frame(videoWidth, videoHeight, rotationDegree,
- strides, null, 0);
- textureFrameToRender = new I420Frame(videoWidth, videoHeight, rotationDegree,
- null, -1, 0);
updateTextureProperties = true;
Log.d(TAG, " YuvImageRenderer.setSize done.");
}
@@ -377,16 +356,8 @@
Log.d(TAG, "ID: " + id + ". Reporting first rendered frame.");
rendererEvents.onFirstFrameRendered();
}
- setSize(frame.width, frame.height, frame.rotationDegree);
- long now = System.nanoTime();
framesReceived++;
- synchronized (frameToRenderQueue) {
- // Skip rendering of this frame if setSize() was not called.
- if (yuvFrameToRender == null || textureFrameToRender == null) {
- framesDropped++;
- VideoRenderer.renderFrameDone(frame);
- return;
- }
+ synchronized (pendingFrameLock) {
// Check input frame parameters.
if (frame.yuvFrame) {
if (frame.yuvStrides[0] < frame.width ||
@@ -397,35 +368,18 @@
VideoRenderer.renderFrameDone(frame);
return;
}
- // Check incoming frame dimensions.
- if (frame.width != yuvFrameToRender.width ||
- frame.height != yuvFrameToRender.height) {
- throw new RuntimeException("Wrong frame size " +
- frame.width + " x " + frame.height);
- }
}
- if (frameToRenderQueue.size() > 0) {
+ if (pendingFrame != null) {
// Skip rendering of this frame if previous frame was not rendered yet.
framesDropped++;
VideoRenderer.renderFrameDone(frame);
return;
}
-
- // Create a local copy of the frame.
- if (frame.yuvFrame) {
- yuvFrameToRender.copyFrom(frame);
- rendererType = RendererType.RENDERER_YUV;
- frameToRenderQueue.offer(yuvFrameToRender);
- } else {
- textureFrameToRender.copyFrom(frame);
- rendererType = RendererType.RENDERER_TEXTURE;
- frameToRenderQueue.offer(textureFrameToRender);
- }
+ pendingFrame = frame;
}
- copyTimeNs += (System.nanoTime() - now);
+ setSize(frame.width, frame.height, frame.rotationDegree);
seenFrame = true;
- VideoRenderer.renderFrameDone(frame);
// Request rendering.
surface.requestRender();
diff --git a/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java b/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java
index a7b88ba..afda848 100644
--- a/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java
+++ b/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java
@@ -37,7 +37,7 @@
*/
public class VideoRenderer {
- /** Java version of cricket::VideoFrame. */
+ /** Java version of cricket::VideoFrame. Frames are only constructed from native code. */
public static class I420Frame {
public final int width;
public final int height;
@@ -46,7 +46,7 @@
public final boolean yuvFrame;
public Object textureObject;
public int textureId;
- // If |nativeFramePointer| is non-zero, the memory is allocated on the C++ side.
+ // Frame pointer in C++.
private long nativeFramePointer;
// rotationDegree is the degree that the frame must be rotated clockwisely
@@ -54,22 +54,14 @@
public int rotationDegree;
/**
- * Construct a frame of the given dimensions with the specified planar
- * data. If |yuvPlanes| is null, new planes of the appropriate sizes are
- * allocated.
+ * Construct a frame of the given dimensions with the specified planar data.
*/
- public I420Frame(
+ private I420Frame(
int width, int height, int rotationDegree,
int[] yuvStrides, ByteBuffer[] yuvPlanes, long nativeFramePointer) {
this.width = width;
this.height = height;
this.yuvStrides = yuvStrides;
- if (yuvPlanes == null) {
- yuvPlanes = new ByteBuffer[3];
- yuvPlanes[0] = ByteBuffer.allocateDirect(yuvStrides[0] * height);
- yuvPlanes[1] = ByteBuffer.allocateDirect(yuvStrides[1] * height / 2);
- yuvPlanes[2] = ByteBuffer.allocateDirect(yuvStrides[2] * height / 2);
- }
this.yuvPlanes = yuvPlanes;
this.yuvFrame = true;
this.rotationDegree = rotationDegree;
@@ -82,7 +74,7 @@
/**
* Construct a texture frame of the given dimensions with data in SurfaceTexture
*/
- public I420Frame(
+ private I420Frame(
int width, int height, int rotationDegree,
Object textureObject, int textureId, long nativeFramePointer) {
this.width = width;
@@ -107,66 +99,6 @@
return (rotationDegree % 180 == 0) ? height : width;
}
- /**
- * Copy the planes out of |source| into |this| and return |this|. Calling
- * this with mismatched frame dimensions or frame type is a programming
- * error and will likely crash.
- */
- public I420Frame copyFrom(I420Frame source) {
- // |nativeFramePointer| is not copied from |source|, because resources in this object are
- // still allocated in Java. After copyFrom() is done, this object should not hold any
- // references to |source|. This is violated for texture frames however, because |textureId|
- // is copied without making a deep copy.
- if (this.nativeFramePointer != 0) {
- throw new RuntimeException("Trying to overwrite a frame allocated in C++");
- }
- if (source.yuvFrame && yuvFrame) {
- if (width != source.width || height != source.height) {
- throw new RuntimeException("Mismatched dimensions! Source: " +
- source.toString() + ", destination: " + toString());
- }
- nativeCopyPlane(source.yuvPlanes[0], width, height,
- source.yuvStrides[0], yuvPlanes[0], yuvStrides[0]);
- nativeCopyPlane(source.yuvPlanes[1], width / 2, height / 2,
- source.yuvStrides[1], yuvPlanes[1], yuvStrides[1]);
- nativeCopyPlane(source.yuvPlanes[2], width / 2, height / 2,
- source.yuvStrides[2], yuvPlanes[2], yuvStrides[2]);
- rotationDegree = source.rotationDegree;
- return this;
- } else if (!source.yuvFrame && !yuvFrame) {
- textureObject = source.textureObject;
- textureId = source.textureId;
- rotationDegree = source.rotationDegree;
- return this;
- } else {
- throw new RuntimeException("Mismatched frame types! Source: " +
- source.toString() + ", destination: " + toString());
- }
- }
-
- public I420Frame copyFrom(byte[] yuvData, int rotationDegree) {
- if (yuvData.length < width * height * 3 / 2) {
- throw new RuntimeException("Wrong arrays size: " + yuvData.length);
- }
- if (!yuvFrame) {
- throw new RuntimeException("Can not feed yuv data to texture frame");
- }
- int planeSize = width * height;
- ByteBuffer[] planes = new ByteBuffer[3];
- planes[0] = ByteBuffer.wrap(yuvData, 0, planeSize);
- planes[1] = ByteBuffer.wrap(yuvData, planeSize, planeSize / 4);
- planes[2] = ByteBuffer.wrap(yuvData, planeSize + planeSize / 4,
- planeSize / 4);
- for (int i = 0; i < 3; i++) {
- yuvPlanes[i].position(0);
- yuvPlanes[i].put(planes[i]);
- yuvPlanes[i].position(0);
- yuvPlanes[i].limit(yuvPlanes[i].capacity());
- }
- this.rotationDegree = rotationDegree;
- return this;
- }
-
@Override
public String toString() {
return width + "x" + height + ":" + yuvStrides[0] + ":" + yuvStrides[1] +
@@ -175,7 +107,7 @@
}
// Helper native function to do a video frame plane copying.
- private static native void nativeCopyPlane(ByteBuffer src, int width,
+ public static native void nativeCopyPlane(ByteBuffer src, int width,
int height, int srcStride, ByteBuffer dst, int dstStride);
/** The real meat of VideoRendererInterface. */
