Android: Split out EGL rendering from SurfaceViewRenderer to separate class
The purpose is to prepare for a TextureViewRenderer that will share the
EGL rendering code.
Two functional changes are also included:
* The implementation of SurfaceHolder.Callback.surfaceDestroyed will now
block until the EGL surface is released. This is done in order to
comply with the documentation that says: "If you have a rendering
thread that directly accesses the surface, you must ensure that thread
is no longer touching the Surface before returning from this function."
* We will no longer try to hide render glitches during layout changes.
This was a lost cause anyway.
BUG=webrtc:6407
Review-Url: https://codereview.webrtc.org/2399463006
Cr-Commit-Position: refs/heads/master@{#14570}
diff --git a/webrtc/api/BUILD.gn b/webrtc/api/BUILD.gn
index 16bbffd..bd70179 100644
--- a/webrtc/api/BUILD.gn
+++ b/webrtc/api/BUILD.gn
@@ -261,6 +261,7 @@
"android/java/src/org/webrtc/EglBase.java",
"android/java/src/org/webrtc/EglBase10.java",
"android/java/src/org/webrtc/EglBase14.java",
+ "android/java/src/org/webrtc/EglRenderer.java",
"android/java/src/org/webrtc/GlRectDrawer.java",
"android/java/src/org/webrtc/GlShader.java",
"android/java/src/org/webrtc/GlTextureFrameBuffer.java",
diff --git a/webrtc/api/android/java/src/org/webrtc/EglRenderer.java b/webrtc/api/android/java/src/org/webrtc/EglRenderer.java
new file mode 100644
index 0000000..f5b9198
--- /dev/null
+++ b/webrtc/api/android/java/src/org/webrtc/EglRenderer.java
@@ -0,0 +1,413 @@
+/*
+ * Copyright 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import android.opengl.GLES20;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.Looper;
+import android.view.Surface;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on an EGL Surface.
+ * This class is intended to be used as a helper class for rendering on SurfaceViews and
+ * TextureViews.
+ */
+public class EglRenderer implements VideoRenderer.Callbacks {
+ private static final String TAG = "EglRenderer";
+ private static final int MAX_SURFACE_CLEAR_COUNT = 3;
+
+ private class EglSurfaceCreation implements Runnable {
+ private Surface surface;
+
+ public synchronized void setSurface(Surface surface) {
+ this.surface = surface;
+ }
+
+ @Override
+ public synchronized void run() {
+ if (surface != null && eglBase != null && !eglBase.hasSurface()) {
+ eglBase.createSurface((Surface) surface);
+ eglBase.makeCurrent();
+ // Necessary for YUV frames with odd width.
+ GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
+ }
+ }
+ }
+
+ private final String name;
+
+ // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
+ // on |handlerLock|.
+ private final Object handlerLock = new Object();
+ private Handler renderThreadHandler;
+
+ // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
+ // from the render thread.
+ private EglBase eglBase;
+ private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader();
+ private RendererCommon.GlDrawer drawer;
+ // Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
+ private int[] yuvTextures = null;
+
+ // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
+ private final Object frameLock = new Object();
+ private VideoRenderer.I420Frame pendingFrame;
+
+ // These variables are synchronized on |layoutLock|.
+ private final Object layoutLock = new Object();
+ private int surfaceWidth;
+ private int surfaceHeight;
+ private float layoutAspectRatio;
+ // If true, mirrors the video stream horizontally.
+ private boolean mirror;
+
+ // These variables are synchronized on |statisticsLock|.
+ private final Object statisticsLock = new Object();
+ // Total number of video frames received in renderFrame() call.
+ private int framesReceived;
+ // Number of video frames dropped by renderFrame() because previous frame has not been rendered
+ // yet.
+ private int framesDropped;
+ // Number of rendered video frames.
+ private int framesRendered;
+ // Time in ns when the first video frame was rendered.
+ private long firstFrameTimeNs;
+ // Time in ns spent in renderFrameOnRenderThread() function.
+ private long renderTimeNs;
+
+ // Runnable for posting frames to render thread.
+ private final Runnable renderFrameRunnable = new Runnable() {
+ @Override
+ public void run() {
+ renderFrameOnRenderThread();
+ }
+ };
+
+ private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation();
+
+ /**
+ * Standard constructor. The name will be used for the render thread name and included when
+ * logging. In order to render something, you must first call init() and createEglSurface.
+ */
+ public EglRenderer(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used
+ * for drawing frames on the EGLSurface. This class is responsible for calling release() on
+ * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous
+ * init()/release() cycle.
+ */
+ public void init(final EglBase.Context sharedContext, final int[] configAttributes,
+ RendererCommon.GlDrawer drawer) {
+ synchronized (handlerLock) {
+ if (renderThreadHandler != null) {
+ throw new IllegalStateException(name + "Already initialized");
+ }
+ logD("Initializing EglRenderer");
+ this.drawer = drawer;
+
+ final HandlerThread renderThread = new HandlerThread(name + "EglRenderer");
+ renderThread.start();
+ renderThreadHandler = new Handler(renderThread.getLooper());
+ // Create EGL context on the newly created render thread. It should be possibly to create the
+ // context on this thread and make it current on the render thread, but this causes failure on
+ // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
+ ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, new Runnable() {
+ @Override
+ public void run() {
+ eglBase = EglBase.create(sharedContext, configAttributes);
+ }
+ });
+ }
+ }
+
+ public void createEglSurface(Surface surface) {
+ eglSurfaceCreationRunnable.setSurface(surface);
+ runOnRenderThread(eglSurfaceCreationRunnable);
+ }
+
+ /**
+ * Block until any pending frame is returned and all GL resources released, even if an interrupt
+ * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function
+ * should be called before the Activity is destroyed and the EGLContext is still valid. If you
+ * don't call this function, the GL resources might leak.
+ */
+ public void release() {
+ final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
+ synchronized (handlerLock) {
+ if (renderThreadHandler == null) {
+ logD("Already released");
+ return;
+ }
+ // Release EGL and GL resources on render thread.
+ renderThreadHandler.postAtFrontOfQueue(new Runnable() {
+ @Override
+ public void run() {
+ if (drawer != null) {
+ drawer.release();
+ drawer = null;
+ }
+ if (yuvTextures != null) {
+ GLES20.glDeleteTextures(3, yuvTextures, 0);
+ yuvTextures = null;
+ }
+ if (eglBase != null) {
+ logD("eglBase detach and release.");
+ eglBase.detachCurrent();
+ eglBase.release();
+ eglBase = null;
+ }
+ eglCleanupBarrier.countDown();
+ }
+ });
+ final Looper renderLooper = renderThreadHandler.getLooper();
+ // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18.
+ renderThreadHandler.post(new Runnable() {
+ @Override
+ public void run() {
+ logD("Quitting render thread.");
+ renderLooper.quit();
+ }
+ });
+ // Don't accept any more frames or messages to the render thread.
+ renderThreadHandler = null;
+ }
+ // Make sure the EGL/GL cleanup posted above is executed.
+ ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
+ synchronized (frameLock) {
+ if (pendingFrame != null) {
+ VideoRenderer.renderFrameDone(pendingFrame);
+ pendingFrame = null;
+ }
+ }
+ resetStatistics();
+ logD("Releasing done.");
+ }
+
+ /**
+ * Reset statistics. This will reset the logged statistics in logStatistics(), and
+ * RendererEvents.onFirstFrameRendered() will be called for the next frame.
+ */
+ public void resetStatistics() {
+ synchronized (statisticsLock) {
+ framesReceived = 0;
+ framesDropped = 0;
+ framesRendered = 0;
+ firstFrameTimeNs = 0;
+ renderTimeNs = 0;
+ }
+ }
+
+ /**
+ * Set if the video stream should be mirrored or not.
+ */
+ public void setMirror(final boolean mirror) {
+ logD("setMirror: " + mirror);
+ synchronized (layoutLock) {
+ this.mirror = mirror;
+ }
+ }
+
+ /**
+ * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video.
+ * Set this to 0 to disable cropping.
+ */
+ public void setLayoutAspectRatio(float layoutAspectRatio) {
+ logD("setLayoutAspectRatio: " + layoutAspectRatio);
+ synchronized (layoutLock) {
+ this.layoutAspectRatio = layoutAspectRatio;
+ }
+ }
+
+ // VideoRenderer.Callbacks interface.
+ @Override
+ public void renderFrame(VideoRenderer.I420Frame frame) {
+ synchronized (statisticsLock) {
+ ++framesReceived;
+ }
+ synchronized (handlerLock) {
+ if (renderThreadHandler == null) {
+ logD("Dropping frame - Not initialized or already released.");
+ VideoRenderer.renderFrameDone(frame);
+ return;
+ }
+ synchronized (frameLock) {
+ if (pendingFrame != null) {
+ // Drop old frame.
+ synchronized (statisticsLock) {
+ ++framesDropped;
+ }
+ VideoRenderer.renderFrameDone(pendingFrame);
+ }
+ pendingFrame = frame;
+ renderThreadHandler.post(renderFrameRunnable);
+ }
+ }
+ }
+
+ /**
+ * Release EGL surface. This function will block until the EGL surface is released.
+ */
+ public void releaseEglSurface() {
+ // Ensure that the render thread is no longer touching the Surface before returning from this
+ // function.
+ eglSurfaceCreationRunnable.setSurface(null /* surface */);
+ synchronized (handlerLock) {
+ if (renderThreadHandler != null) {
+ renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable);
+ ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, new Runnable() {
+ @Override
+ public void run() {
+ if (eglBase != null) {
+ eglBase.detachCurrent();
+ eglBase.releaseSurface();
+ }
+ }
+ });
+ }
+ }
+ }
+
+ /**
+ * Notify that the surface size has changed.
+ */
+ public void surfaceSizeChanged(int surfaceWidth, int surfaceHeight) {
+ logD("Surface size changed: " + surfaceWidth + "x" + surfaceHeight);
+ synchronized (layoutLock) {
+ this.surfaceWidth = surfaceWidth;
+ this.surfaceHeight = surfaceHeight;
+ }
+ }
+
+ /**
+ * Private helper function to post tasks safely.
+ */
+ private void runOnRenderThread(Runnable runnable) {
+ synchronized (handlerLock) {
+ if (renderThreadHandler != null) {
+ renderThreadHandler.post(runnable);
+ }
+ }
+ }
+
+ private void makeBlack() {
+ if (eglBase != null && eglBase.hasSurface()) {
+ logD("clearSurface");
+ GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+ eglBase.swapBuffers();
+ }
+ }
+
+ /**
+ * Renders and releases |pendingFrame|.
+ */
+ private void renderFrameOnRenderThread() {
+ // Fetch and render |pendingFrame|.
+ final VideoRenderer.I420Frame frame;
+ synchronized (frameLock) {
+ if (pendingFrame == null) {
+ return;
+ }
+ frame = pendingFrame;
+ pendingFrame = null;
+ }
+ if (eglBase == null || !eglBase.hasSurface()) {
+ logD("Dropping frame - No surface");
+ VideoRenderer.renderFrameDone(frame);
+ return;
+ }
+
+ final long startTimeNs = System.nanoTime();
+ float[] texMatrix =
+ RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
+
+ // After a surface size change, the EGLSurface might still have a buffer of the old size in the
+ // pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
+ // changed. Such a buffer will be rendered incorrectly, so flush it with a black frame.
+ synchronized (layoutLock) {
+ int surfaceClearCount = 0;
+ while (eglBase.surfaceWidth() != surfaceWidth || eglBase.surfaceHeight() != surfaceHeight) {
+ ++surfaceClearCount;
+ if (surfaceClearCount > MAX_SURFACE_CLEAR_COUNT) {
+ logD("Failed to get surface of expected size - dropping frame.");
+ VideoRenderer.renderFrameDone(frame);
+ return;
+ }
+ logD("Surface size mismatch - clearing surface.");
+ makeBlack();
+ }
+ final float[] layoutMatrix;
+ if (layoutAspectRatio > 0) {
+ layoutMatrix = RendererCommon.getLayoutMatrix(
+ mirror, frame.rotatedWidth() / (float) frame.rotatedHeight(), layoutAspectRatio);
+ } else {
+ layoutMatrix =
+ mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix();
+ }
+ texMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
+ }
+
+ GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+ if (frame.yuvFrame) {
+ // Make sure YUV textures are allocated.
+ if (yuvTextures == null) {
+ yuvTextures = new int[3];
+ for (int i = 0; i < 3; i++) {
+ yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
+ }
+ }
+ yuvUploader.uploadYuvData(
+ yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
+ drawer.drawYuv(yuvTextures, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+ surfaceWidth, surfaceHeight);
+ } else {
+ drawer.drawOes(frame.textureId, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+ surfaceWidth, surfaceHeight);
+ }
+
+ eglBase.swapBuffers();
+ VideoRenderer.renderFrameDone(frame);
+ synchronized (statisticsLock) {
+ if (framesRendered == 0) {
+ firstFrameTimeNs = startTimeNs;
+ }
+ ++framesRendered;
+ renderTimeNs += (System.nanoTime() - startTimeNs);
+ if (framesRendered % 300 == 0) {
+ logStatistics();
+ }
+ }
+ }
+
+ private void logStatistics() {
+ synchronized (statisticsLock) {
+ logD("Frames received: " + framesReceived + ". Dropped: " + framesDropped + ". Rendered: "
+ + framesRendered);
+ if (framesReceived > 0 && framesRendered > 0) {
+ final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs;
+ logD("Duration: " + (int) (timeSinceFirstFrameNs / 1e6) + " ms. FPS: "
+ + framesRendered * 1e9 / timeSinceFirstFrameNs);
+ logD("Average render time: " + (int) (renderTimeNs / (1000 * framesRendered)) + " us.");
+ }
+ }
+ }
+
+ private void logD(String string) {
+ Logging.d(TAG, name + string);
+ }
+}
diff --git a/webrtc/api/android/java/src/org/webrtc/RendererCommon.java b/webrtc/api/android/java/src/org/webrtc/RendererCommon.java
index 8eb0461..a82cea1 100644
--- a/webrtc/api/android/java/src/org/webrtc/RendererCommon.java
+++ b/webrtc/api/android/java/src/org/webrtc/RendererCommon.java
@@ -137,11 +137,10 @@
// and maximum layout size.
final float frameAspect = frameWidth / (float) frameHeight;
final float displayAspect = maxWidth / (float) maxHeight;
- final RendererCommon.ScalingType scalingType = (frameAspect > 1.0f) == (displayAspect > 1.0f)
+ final ScalingType scalingType = (frameAspect > 1.0f) == (displayAspect > 1.0f)
? scalingTypeMatchOrientation
: scalingTypeMismatchOrientation;
- final Point layoutSize =
- RendererCommon.getDisplaySize(scalingType, frameAspect, maxWidth, maxHeight);
+ final Point layoutSize = getDisplaySize(scalingType, frameAspect, maxWidth, maxHeight);
// If the measure specification is forcing a specific size - yield.
if (View.MeasureSpec.getMode(widthSpec) == View.MeasureSpec.EXACTLY) {
diff --git a/webrtc/api/android/java/src/org/webrtc/SurfaceViewRenderer.java b/webrtc/api/android/java/src/org/webrtc/SurfaceViewRenderer.java
index 5b19c43..428198d 100644
--- a/webrtc/api/android/java/src/org/webrtc/SurfaceViewRenderer.java
+++ b/webrtc/api/android/java/src/org/webrtc/SurfaceViewRenderer.java
@@ -13,17 +13,10 @@
import android.content.Context;
import android.content.res.Resources.NotFoundException;
import android.graphics.Point;
-import android.opengl.GLES20;
-import android.os.Handler;
-import android.os.HandlerThread;
import android.util.AttributeSet;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
-import java.util.concurrent.CountDownLatch;
-
-import javax.microedition.khronos.egl.EGLContext;
-
/**
* Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on a SurfaceView.
* renderFrame() is asynchronous to avoid blocking the calling thread.
@@ -37,91 +30,28 @@
extends SurfaceView implements SurfaceHolder.Callback, VideoRenderer.Callbacks {
private static final String TAG = "SurfaceViewRenderer";
- // Dedicated render thread.
- private HandlerThread renderThread;
- // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
- // on |handlerLock|.
- private final Object handlerLock = new Object();
- private Handler renderThreadHandler;
+ // Cached resource name.
+ private final String resourceName;
+ private final RendererCommon.VideoLayoutMeasure videoLayoutMeasure =
+ new RendererCommon.VideoLayoutMeasure();
+ private final EglRenderer eglRenderer;
- // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
- // from the render thread.
- private EglBase eglBase;
- private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader();
- private RendererCommon.GlDrawer drawer;
- // Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
- private int[] yuvTextures = null;
+ // Callback for reporting renderer events. Read-only after initilization so no lock required.
+ private RendererCommon.RendererEvents rendererEvents;
- // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
- private final Object frameLock = new Object();
- private VideoRenderer.I420Frame pendingFrame;
-
- // These variables are synchronized on |layoutLock|.
private final Object layoutLock = new Object();
- // These dimension values are used to keep track of the state in these functions: onMeasure(),
- // onLayout(), and surfaceChanged(). A new layout is triggered with requestLayout(). This happens
- // internally when the incoming frame size changes. requestLayout() can also be triggered
- // externally. The layout change is a two pass process: first onMeasure() is called in a top-down
- // traversal of the View tree, followed by an onLayout() pass that is also top-down. During the
- // onLayout() pass, each parent is responsible for positioning its children using the sizes
- // computed in the measure pass.
- // |desiredLayoutsize| is the layout size we have requested in onMeasure() and are waiting for to
- // take effect.
- private Point desiredLayoutSize = new Point();
- // |layoutSize|/|surfaceSize| is the actual current layout/surface size. They are updated in
- // onLayout() and surfaceChanged() respectively.
- private final Point layoutSize = new Point();
- // TODO(magjed): Enable hardware scaler with SurfaceHolder.setFixedSize(). This will decouple
- // layout and surface size.
- private final Point surfaceSize = new Point();
- // |isSurfaceCreated| keeps track of the current status in surfaceCreated()/surfaceDestroyed().
- private boolean isSurfaceCreated;
- // Last rendered frame dimensions, or 0 if no frame has been rendered yet.
+ private boolean isFirstFrameRendered;
private int rotatedFrameWidth;
private int rotatedFrameHeight;
private int frameRotation;
- private final RendererCommon.VideoLayoutMeasure videoLayoutMeasure =
- new RendererCommon.VideoLayoutMeasure();
-
- // If true, mirrors the video stream horizontally.
- private boolean mirror;
- // Callback for reporting renderer events.
- private RendererCommon.RendererEvents rendererEvents;
-
- // These variables are synchronized on |statisticsLock|.
- private final Object statisticsLock = new Object();
- // Total number of video frames received in renderFrame() call.
- private int framesReceived;
- // Number of video frames dropped by renderFrame() because previous frame has not been rendered
- // yet.
- private int framesDropped;
- // Number of rendered video frames.
- private int framesRendered;
- // Time in ns when the first video frame was rendered.
- private long firstFrameTimeNs;
- // Time in ns spent in renderFrameOnRenderThread() function.
- private long renderTimeNs;
-
- // Runnable for posting frames to render thread.
- private final Runnable renderFrameRunnable = new Runnable() {
- @Override
- public void run() {
- renderFrameOnRenderThread();
- }
- };
- // Runnable for clearing Surface to black.
- private final Runnable makeBlackRunnable = new Runnable() {
- @Override
- public void run() {
- makeBlack();
- }
- };
/**
* Standard View constructor. In order to render something, you must first call init().
*/
public SurfaceViewRenderer(Context context) {
super(context);
+ this.resourceName = getResourceName();
+ eglRenderer = new EglRenderer(resourceName);
getHolder().addCallback(this);
}
@@ -130,6 +60,8 @@
*/
public SurfaceViewRenderer(Context context, AttributeSet attrs) {
super(context, attrs);
+ this.resourceName = getResourceName();
+ eglRenderer = new EglRenderer(resourceName);
getHolder().addCallback(this);
}
@@ -150,48 +82,14 @@
public void init(final EglBase.Context sharedContext,
RendererCommon.RendererEvents rendererEvents, final int[] configAttributes,
RendererCommon.GlDrawer drawer) {
- synchronized (handlerLock) {
- if (renderThreadHandler != null) {
- throw new IllegalStateException(getResourceName() + "Already initialized");
- }
- Logging.d(TAG, getResourceName() + "Initializing.");
- this.rendererEvents = rendererEvents;
- this.drawer = drawer;
- renderThread = new HandlerThread(TAG);
- renderThread.start();
- renderThreadHandler = new Handler(renderThread.getLooper());
- // Create EGL context on the newly created render thread. It should be possibly to create the
- // context on this thread and make it current on the render thread, but this causes failure on
- // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
- ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, new Runnable() {
- @Override
- public void run() {
- eglBase = EglBase.create(sharedContext, configAttributes);
- }
- });
+ ThreadUtils.checkIsOnMainThread();
+ this.rendererEvents = rendererEvents;
+ synchronized (layoutLock) {
+ rotatedFrameWidth = 0;
+ rotatedFrameHeight = 0;
+ frameRotation = 0;
}
- tryCreateEglSurface();
- }
-
- /**
- * Create and make an EGLSurface current if both init() and surfaceCreated() have been called.
- */
- public void tryCreateEglSurface() {
- // |renderThreadHandler| is only created after |eglBase| is created in init(), so the
- // following code will only execute if eglBase != null.
- runOnRenderThread(new Runnable() {
- @Override
- public void run() {
- synchronized (layoutLock) {
- if (eglBase != null && isSurfaceCreated && !eglBase.hasSurface()) {
- eglBase.createSurface(getHolder().getSurface());
- eglBase.makeCurrent();
- // Necessary for YUV frames with odd width.
- GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
- }
- }
- }
- });
+ eglRenderer.init(sharedContext, configAttributes, drawer);
}
/**
@@ -201,78 +99,14 @@
* don't call this function, the GL resources might leak.
*/
public void release() {
- final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
- synchronized (handlerLock) {
- if (renderThreadHandler == null) {
- Logging.d(TAG, getResourceName() + "Already released");
- return;
- }
- // Release EGL and GL resources on render thread.
- // TODO(magjed): This might not be necessary - all OpenGL resources are automatically deleted
- // when the EGL context is lost. It might be dangerous to delete them manually in
- // Activity.onDestroy().
- renderThreadHandler.postAtFrontOfQueue(new Runnable() {
- @Override
- public void run() {
- drawer.release();
- drawer = null;
- if (yuvTextures != null) {
- GLES20.glDeleteTextures(3, yuvTextures, 0);
- yuvTextures = null;
- }
- // Clear last rendered image to black.
- makeBlack();
- eglBase.release();
- eglBase = null;
- eglCleanupBarrier.countDown();
- }
- });
- // Don't accept any more frames or messages to the render thread.
- renderThreadHandler = null;
- }
- // Make sure the EGL/GL cleanup posted above is executed.
- ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
- renderThread.quit();
- synchronized (frameLock) {
- if (pendingFrame != null) {
- VideoRenderer.renderFrameDone(pendingFrame);
- pendingFrame = null;
- }
- }
- // The |renderThread| cleanup is not safe to cancel and we need to wait until it's done.
- ThreadUtils.joinUninterruptibly(renderThread);
- renderThread = null;
- // Reset statistics and event reporting.
- synchronized (layoutLock) {
- rotatedFrameWidth = 0;
- rotatedFrameHeight = 0;
- frameRotation = 0;
- rendererEvents = null;
- }
- resetStatistics();
- }
-
- /**
- * Reset statistics. This will reset the logged statistics in logStatistics(), and
- * RendererEvents.onFirstFrameRendered() will be called for the next frame.
- */
- public void resetStatistics() {
- synchronized (statisticsLock) {
- framesReceived = 0;
- framesDropped = 0;
- framesRendered = 0;
- firstFrameTimeNs = 0;
- renderTimeNs = 0;
- }
+ eglRenderer.release();
}
/**
* Set if the video stream should be mirrored or not.
*/
public void setMirror(final boolean mirror) {
- synchronized (layoutLock) {
- this.mirror = mirror;
- }
+ eglRenderer.setMirror(mirror);
}
/**
@@ -292,112 +126,46 @@
// VideoRenderer.Callbacks interface.
@Override
public void renderFrame(VideoRenderer.I420Frame frame) {
- synchronized (statisticsLock) {
- ++framesReceived;
- }
- synchronized (handlerLock) {
- if (renderThreadHandler == null) {
- Logging.d(TAG, getResourceName() + "Dropping frame - Not initialized or already released.");
- VideoRenderer.renderFrameDone(frame);
- return;
- }
- synchronized (frameLock) {
- if (pendingFrame != null) {
- // Drop old frame.
- synchronized (statisticsLock) {
- ++framesDropped;
- }
- VideoRenderer.renderFrameDone(pendingFrame);
- }
- pendingFrame = frame;
- renderThreadHandler.post(renderFrameRunnable);
- }
- }
+ updateFrameDimensionsAndReportEvents(frame);
+ eglRenderer.renderFrame(frame);
}
// View layout interface.
@Override
protected void onMeasure(int widthSpec, int heightSpec) {
ThreadUtils.checkIsOnMainThread();
- final boolean isNewSize;
+ final Point size;
synchronized (layoutLock) {
- desiredLayoutSize =
+ size =
videoLayoutMeasure.measure(widthSpec, heightSpec, rotatedFrameWidth, rotatedFrameHeight);
- isNewSize =
- (desiredLayoutSize.x != getMeasuredWidth() || desiredLayoutSize.y != getMeasuredHeight());
- setMeasuredDimension(desiredLayoutSize.x, desiredLayoutSize.y);
}
- if (isNewSize) {
- // Clear the surface asap before the layout change to avoid stretched video and other
- // render artifacs. Don't wait for it to finish because the IO thread should never be
- // blocked, so it's a best-effort attempt.
- synchronized (handlerLock) {
- if (renderThreadHandler != null) {
- renderThreadHandler.postAtFrontOfQueue(makeBlackRunnable);
- }
- }
- }
+ setMeasuredDimension(size.x, size.y);
+ logD("onMeasure(). New size: " + size.x + "x" + size.y);
}
@Override
protected void onLayout(boolean changed, int left, int top, int right, int bottom) {
- synchronized (layoutLock) {
- layoutSize.x = right - left;
- layoutSize.y = bottom - top;
- }
- // Might have a pending frame waiting for a layout of correct size.
- runOnRenderThread(renderFrameRunnable);
+ ThreadUtils.checkIsOnMainThread();
+ eglRenderer.setLayoutAspectRatio((right - left) / (float) (bottom - top));
}
// SurfaceHolder.Callback interface.
@Override
public void surfaceCreated(final SurfaceHolder holder) {
- Logging.d(TAG, getResourceName() + "Surface created.");
- synchronized (layoutLock) {
- isSurfaceCreated = true;
- }
- tryCreateEglSurface();
+ ThreadUtils.checkIsOnMainThread();
+ eglRenderer.createEglSurface(holder.getSurface());
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
- Logging.d(TAG, getResourceName() + "Surface destroyed.");
- synchronized (layoutLock) {
- isSurfaceCreated = false;
- surfaceSize.x = 0;
- surfaceSize.y = 0;
- }
- runOnRenderThread(new Runnable() {
- @Override
- public void run() {
- if (eglBase != null) {
- eglBase.detachCurrent();
- eglBase.releaseSurface();
- }
- }
- });
+ ThreadUtils.checkIsOnMainThread();
+ eglRenderer.releaseEglSurface();
}
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
- Logging.d(TAG, getResourceName() + "Surface changed: " + width + "x" + height);
- synchronized (layoutLock) {
- surfaceSize.x = width;
- surfaceSize.y = height;
- }
- // Might have a pending frame waiting for a surface of correct size.
- runOnRenderThread(renderFrameRunnable);
- }
-
- /**
- * Private helper function to post tasks safely.
- */
- private void runOnRenderThread(Runnable runnable) {
- synchronized (handlerLock) {
- if (renderThreadHandler != null) {
- renderThreadHandler.post(runnable);
- }
- }
+ ThreadUtils.checkIsOnMainThread();
+ eglRenderer.surfaceSizeChanged(width, height);
}
private String getResourceName() {
@@ -408,125 +176,20 @@
}
}
- private void makeBlack() {
- if (Thread.currentThread() != renderThread) {
- throw new IllegalStateException(getResourceName() + "Wrong thread.");
- }
- if (eglBase != null && eglBase.hasSurface()) {
- GLES20.glClearColor(0, 0, 0, 0);
- GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
- eglBase.swapBuffers();
- }
- }
-
- /**
- * Requests new layout if necessary. Returns true if layout and surface size are consistent.
- */
- private boolean checkConsistentLayout() {
- if (Thread.currentThread() != renderThread) {
- throw new IllegalStateException(getResourceName() + "Wrong thread.");
- }
- synchronized (layoutLock) {
- // Return false while we are in the middle of a layout change.
- return layoutSize.equals(desiredLayoutSize) && surfaceSize.equals(layoutSize);
- }
- }
-
- /**
- * Renders and releases |pendingFrame|.
- */
- private void renderFrameOnRenderThread() {
- if (Thread.currentThread() != renderThread) {
- throw new IllegalStateException(getResourceName() + "Wrong thread.");
- }
- // Fetch and render |pendingFrame|.
- final VideoRenderer.I420Frame frame;
- synchronized (frameLock) {
- if (pendingFrame == null) {
- return;
- }
- frame = pendingFrame;
- pendingFrame = null;
- }
- updateFrameDimensionsAndReportEvents(frame);
- if (eglBase == null || !eglBase.hasSurface()) {
- Logging.d(TAG, getResourceName() + "No surface to draw on");
- VideoRenderer.renderFrameDone(frame);
- return;
- }
- if (!checkConsistentLayout()) {
- // Output intermediate black frames while the layout is updated.
- makeBlack();
- VideoRenderer.renderFrameDone(frame);
- return;
- }
- // After a surface size change, the EGLSurface might still have a buffer of the old size in the
- // pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
- // changed. Such a buffer will be rendered incorrectly, so flush it with a black frame.
- synchronized (layoutLock) {
- if (eglBase.surfaceWidth() != surfaceSize.x || eglBase.surfaceHeight() != surfaceSize.y) {
- makeBlack();
- }
- }
-
- final long startTimeNs = System.nanoTime();
- final float[] texMatrix;
- synchronized (layoutLock) {
- final float[] rotatedSamplingMatrix =
- RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
- final float[] layoutMatrix = RendererCommon.getLayoutMatrix(mirror,
- frame.rotatedWidth() / (float) frame.rotatedHeight(),
- layoutSize.x / (float) layoutSize.y);
- texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix);
- }
-
- // TODO(magjed): glClear() shouldn't be necessary since every pixel is covered anyway, but it's
- // a workaround for bug 5147. Performance will be slightly worse.
- GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
- if (frame.yuvFrame) {
- // Make sure YUV textures are allocated.
- if (yuvTextures == null) {
- yuvTextures = new int[3];
- for (int i = 0; i < 3; i++) {
- yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
- }
- }
- yuvUploader.uploadYuvData(
- yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
- drawer.drawYuv(yuvTextures, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
- surfaceSize.x, surfaceSize.y);
- } else {
- drawer.drawOes(frame.textureId, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
- surfaceSize.x, surfaceSize.y);
- }
-
- eglBase.swapBuffers();
- VideoRenderer.renderFrameDone(frame);
- synchronized (statisticsLock) {
- if (framesRendered == 0) {
- firstFrameTimeNs = startTimeNs;
- synchronized (layoutLock) {
- Logging.d(TAG, getResourceName() + "Reporting first rendered frame.");
- if (rendererEvents != null) {
- rendererEvents.onFirstFrameRendered();
- }
- }
- }
- ++framesRendered;
- renderTimeNs += (System.nanoTime() - startTimeNs);
- if (framesRendered % 300 == 0) {
- logStatistics();
- }
- }
- }
-
// Update frame dimensions and report any changes to |rendererEvents|.
private void updateFrameDimensionsAndReportEvents(VideoRenderer.I420Frame frame) {
synchronized (layoutLock) {
+ if (!isFirstFrameRendered) {
+ isFirstFrameRendered = true;
+ logD("Reporting first rendered frame.");
+ if (rendererEvents != null) {
+ rendererEvents.onFirstFrameRendered();
+ }
+ }
if (rotatedFrameWidth != frame.rotatedWidth() || rotatedFrameHeight != frame.rotatedHeight()
|| frameRotation != frame.rotationDegree) {
- Logging.d(TAG, getResourceName() + "Reporting frame resolution changed to " + frame.width
- + "x" + frame.height + " with rotation " + frame.rotationDegree);
+ logD("Reporting frame resolution changed to " + frame.width + "x" + frame.height
+ + " with rotation " + frame.rotationDegree);
if (rendererEvents != null) {
rendererEvents.onFrameResolutionChanged(frame.width, frame.height, frame.rotationDegree);
}
@@ -543,17 +206,7 @@
}
}
- private void logStatistics() {
- synchronized (statisticsLock) {
- Logging.d(TAG, getResourceName() + "Frames received: " + framesReceived + ". Dropped: "
- + framesDropped + ". Rendered: " + framesRendered);
- if (framesReceived > 0 && framesRendered > 0) {
- final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs;
- Logging.d(TAG, getResourceName() + "Duration: " + (int) (timeSinceFirstFrameNs / 1e6)
- + " ms. FPS: " + framesRendered * 1e9 / timeSinceFirstFrameNs);
- Logging.d(TAG, getResourceName() + "Average render time: "
- + (int) (renderTimeNs / (1000 * framesRendered)) + " us.");
- }
- }
+ private void logD(String string) {
+ Logging.d(TAG, resourceName + string);
}
}
