blob: 837bd6ee41de1e84d88560c7f7f151e246f6e1e4 [file] [log] [blame]
/*
* Copyright 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import android.graphics.SurfaceTexture;
import android.opengl.GLES20;
import android.os.SystemClock;
import android.support.test.filters.MediumTest;
import android.support.test.filters.SmallTest;
import java.nio.ByteBuffer;
import java.util.concurrent.CountDownLatch;
import javax.annotation.Nullable;
import org.chromium.base.test.BaseJUnit4ClassRunner;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(BaseJUnit4ClassRunner.class)
public class SurfaceTextureHelperTest {
/**
* Mock texture listener with blocking wait functionality.
*/
public static final class MockTextureListener implements VideoSink {
private final Object lock = new Object();
private @Nullable VideoFrame.TextureBuffer textureBuffer;
// Thread where frames are expected to be received on.
private final @Nullable Thread expectedThread;
MockTextureListener() {
this.expectedThread = null;
}
MockTextureListener(Thread expectedThread) {
this.expectedThread = expectedThread;
}
@Override
public void onFrame(VideoFrame frame) {
if (expectedThread != null && Thread.currentThread() != expectedThread) {
throw new IllegalStateException("onTextureFrameAvailable called on wrong thread.");
}
synchronized (lock) {
this.textureBuffer = (VideoFrame.TextureBuffer) frame.getBuffer();
textureBuffer.retain();
lock.notifyAll();
}
}
/** Wait indefinitely for a new textureBuffer. */
public VideoFrame.TextureBuffer waitForTextureBuffer() throws InterruptedException {
synchronized (lock) {
while (true) {
final VideoFrame.TextureBuffer textureBufferToReturn = textureBuffer;
if (textureBufferToReturn != null) {
textureBuffer = null;
return textureBufferToReturn;
}
lock.wait();
}
}
}
/** Make sure we get no frame in the specified time period. */
public void assertNoFrameIsDelivered(final long waitPeriodMs) throws InterruptedException {
final long startTimeMs = SystemClock.elapsedRealtime();
long timeRemainingMs = waitPeriodMs;
synchronized (lock) {
while (textureBuffer == null && timeRemainingMs > 0) {
lock.wait(timeRemainingMs);
final long elapsedTimeMs = SystemClock.elapsedRealtime() - startTimeMs;
timeRemainingMs = waitPeriodMs - elapsedTimeMs;
}
assertTrue(textureBuffer == null);
}
}
}
/** Assert that two integers are close, with difference at most
* {@code threshold}. */
public static void assertClose(int threshold, int expected, int actual) {
if (Math.abs(expected - actual) <= threshold)
return;
fail("Not close enough, threshold " + threshold + ". Expected: " + expected + " Actual: "
+ actual);
}
@Before
public void setUp() {
// Load the JNI library for textureToYuv.
NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);
}
/**
* Test normal use by receiving three uniform texture frames. Texture frames are returned as early
* as possible. The texture pixel values are inspected by drawing the texture frame to a pixel
* buffer and reading it back with glReadPixels().
*/
@Test
@MediumTest
public void testThreeConstantColorFrames() throws InterruptedException {
final int width = 16;
final int height = 16;
// Create EGL base with a pixel buffer as display output.
final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PIXEL_BUFFER);
eglBase.createPbufferSurface(width, height);
final GlRectDrawer drawer = new GlRectDrawer();
// Create SurfaceTextureHelper and listener.
final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create(
"SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext());
final MockTextureListener listener = new MockTextureListener();
surfaceTextureHelper.startListening(listener);
surfaceTextureHelper.setTextureSize(width, height);
// Create resources for stubbing an OES texture producer. |eglOesBase| has the SurfaceTexture in
// |surfaceTextureHelper| as the target EGLSurface.
final EglBase eglOesBase = EglBase.create(eglBase.getEglBaseContext(), EglBase.CONFIG_PLAIN);
eglOesBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
assertEquals(eglOesBase.surfaceWidth(), width);
assertEquals(eglOesBase.surfaceHeight(), height);
final int red[] = new int[] {79, 144, 185};
final int green[] = new int[] {66, 210, 162};
final int blue[] = new int[] {161, 117, 158};
// Draw three frames.
for (int i = 0; i < 3; ++i) {
// Draw a constant color frame onto the SurfaceTexture.
eglOesBase.makeCurrent();
GLES20.glClearColor(red[i] / 255.0f, green[i] / 255.0f, blue[i] / 255.0f, 1.0f);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// swapBuffers() will ultimately trigger onTextureFrameAvailable().
eglOesBase.swapBuffers();
// Wait for an OES texture to arrive and draw it onto the pixel buffer.
final VideoFrame.TextureBuffer textureBuffer = listener.waitForTextureBuffer();
eglBase.makeCurrent();
drawer.drawOes(textureBuffer.getTextureId(),
RendererCommon.convertMatrixFromAndroidGraphicsMatrix(textureBuffer.getTransformMatrix()),
width, height, 0, 0, width, height);
textureBuffer.release();
// Download the pixels in the pixel buffer as RGBA. Not all platforms support RGB, e.g.
// Nexus 9.
final ByteBuffer rgbaData = ByteBuffer.allocateDirect(width * height * 4);
GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgbaData);
GlUtil.checkNoGLES2Error("glReadPixels");
// Assert rendered image is expected constant color.
while (rgbaData.hasRemaining()) {
assertEquals(rgbaData.get() & 0xFF, red[i]);
assertEquals(rgbaData.get() & 0xFF, green[i]);
assertEquals(rgbaData.get() & 0xFF, blue[i]);
assertEquals(rgbaData.get() & 0xFF, 255);
}
}
drawer.release();
surfaceTextureHelper.dispose();
eglBase.release();
}
/**
* Test disposing the SurfaceTextureHelper while holding a pending texture frame. The pending
* texture frame should still be valid, and this is tested by drawing the texture frame to a pixel
* buffer and reading it back with glReadPixels().
*/
@Test
@MediumTest
public void testLateReturnFrame() throws InterruptedException {
final int width = 16;
final int height = 16;
// Create EGL base with a pixel buffer as display output.
final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PIXEL_BUFFER);
eglBase.createPbufferSurface(width, height);
// Create SurfaceTextureHelper and listener.
final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create(
"SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext());
final MockTextureListener listener = new MockTextureListener();
surfaceTextureHelper.startListening(listener);
surfaceTextureHelper.setTextureSize(width, height);
// Create resources for stubbing an OES texture producer. |eglOesBase| has the SurfaceTexture in
// |surfaceTextureHelper| as the target EGLSurface.
final EglBase eglOesBase = EglBase.create(eglBase.getEglBaseContext(), EglBase.CONFIG_PLAIN);
eglOesBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
assertEquals(eglOesBase.surfaceWidth(), width);
assertEquals(eglOesBase.surfaceHeight(), height);
final int red = 79;
final int green = 66;
final int blue = 161;
// Draw a constant color frame onto the SurfaceTexture.
eglOesBase.makeCurrent();
GLES20.glClearColor(red / 255.0f, green / 255.0f, blue / 255.0f, 1.0f);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// swapBuffers() will ultimately trigger onTextureFrameAvailable().
eglOesBase.swapBuffers();
eglOesBase.release();
// Wait for OES texture frame.
final VideoFrame.TextureBuffer textureBuffer = listener.waitForTextureBuffer();
// Diconnect while holding the frame.
surfaceTextureHelper.dispose();
// Draw the pending texture frame onto the pixel buffer.
eglBase.makeCurrent();
final GlRectDrawer drawer = new GlRectDrawer();
drawer.drawOes(textureBuffer.getTextureId(),
RendererCommon.convertMatrixFromAndroidGraphicsMatrix(textureBuffer.getTransformMatrix()),
width, height, 0, 0, width, height);
drawer.release();
// Download the pixels in the pixel buffer as RGBA. Not all platforms support RGB, e.g. Nexus 9.
final ByteBuffer rgbaData = ByteBuffer.allocateDirect(width * height * 4);
GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgbaData);
GlUtil.checkNoGLES2Error("glReadPixels");
eglBase.release();
// Assert rendered image is expected constant color.
while (rgbaData.hasRemaining()) {
assertEquals(rgbaData.get() & 0xFF, red);
assertEquals(rgbaData.get() & 0xFF, green);
assertEquals(rgbaData.get() & 0xFF, blue);
assertEquals(rgbaData.get() & 0xFF, 255);
}
// Late frame return after everything has been disposed and released.
textureBuffer.release();
}
/**
* Test disposing the SurfaceTextureHelper, but keep trying to produce more texture frames. No
* frames should be delivered to the listener.
*/
@Test
@MediumTest
public void testDispose() throws InterruptedException {
// Create SurfaceTextureHelper and listener.
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
final MockTextureListener listener = new MockTextureListener();
surfaceTextureHelper.startListening(listener);
// Create EglBase with the SurfaceTexture as target EGLSurface.
final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN);
surfaceTextureHelper.setTextureSize(/* textureWidth= */ 32, /* textureHeight= */ 32);
eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
eglBase.makeCurrent();
// Assert no frame has been received yet.
listener.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1);
// Draw and wait for one frame.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// swapBuffers() will ultimately trigger onTextureFrameAvailable().
eglBase.swapBuffers();
listener.waitForTextureBuffer().release();
// Dispose - we should not receive any textures after this.
surfaceTextureHelper.dispose();
// Draw one frame.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
eglBase.swapBuffers();
// swapBuffers() should not trigger onTextureFrameAvailable() because disposed has been called.
// Assert that no OES texture was delivered.
listener.assertNoFrameIsDelivered(/* waitPeriodMs= */ 500);
eglBase.release();
}
/**
* Test disposing the SurfaceTextureHelper immediately after is has been setup to use a
* shared context. No frames should be delivered to the listener.
*/
@Test
@SmallTest
public void testDisposeImmediately() {
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
surfaceTextureHelper.dispose();
}
/**
* Call stopListening(), but keep trying to produce more texture frames. No frames should be
* delivered to the listener.
*/
@Test
@MediumTest
public void testStopListening() throws InterruptedException {
// Create SurfaceTextureHelper and listener.
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
surfaceTextureHelper.setTextureSize(/* textureWidth= */ 32, /* textureHeight= */ 32);
final MockTextureListener listener = new MockTextureListener();
surfaceTextureHelper.startListening(listener);
// Create EglBase with the SurfaceTexture as target EGLSurface.
final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN);
eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
eglBase.makeCurrent();
// Assert no frame has been received yet.
listener.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1);
// Draw and wait for one frame.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// swapBuffers() will ultimately trigger onTextureFrameAvailable().
eglBase.swapBuffers();
listener.waitForTextureBuffer().release();
// Stop listening - we should not receive any textures after this.
surfaceTextureHelper.stopListening();
// Draw one frame.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
eglBase.swapBuffers();
// swapBuffers() should not trigger onTextureFrameAvailable() because disposed has been called.
// Assert that no OES texture was delivered.
listener.assertNoFrameIsDelivered(/* waitPeriodMs= */ 500);
surfaceTextureHelper.dispose();
eglBase.release();
}
/**
* Test stopListening() immediately after the SurfaceTextureHelper has been setup.
*/
@Test
@SmallTest
public void testStopListeningImmediately() throws InterruptedException {
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
final MockTextureListener listener = new MockTextureListener();
surfaceTextureHelper.startListening(listener);
surfaceTextureHelper.stopListening();
surfaceTextureHelper.dispose();
}
/**
* Test stopListening() immediately after the SurfaceTextureHelper has been setup on the handler
* thread.
*/
@Test
@SmallTest
public void testStopListeningImmediatelyOnHandlerThread() throws InterruptedException {
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
final MockTextureListener listener = new MockTextureListener();
final CountDownLatch stopListeningBarrier = new CountDownLatch(1);
final CountDownLatch stopListeningBarrierDone = new CountDownLatch(1);
// Start by posting to the handler thread to keep it occupied.
surfaceTextureHelper.getHandler().post(new Runnable() {
@Override
public void run() {
ThreadUtils.awaitUninterruptibly(stopListeningBarrier);
surfaceTextureHelper.stopListening();
stopListeningBarrierDone.countDown();
}
});
// startListening() is asynchronous and will post to the occupied handler thread.
surfaceTextureHelper.startListening(listener);
// Wait for stopListening() to be called on the handler thread.
stopListeningBarrier.countDown();
stopListeningBarrierDone.await();
// Wait until handler thread is idle to try to catch late startListening() call.
final CountDownLatch barrier = new CountDownLatch(1);
surfaceTextureHelper.getHandler().post(new Runnable() {
@Override
public void run() {
barrier.countDown();
}
});
ThreadUtils.awaitUninterruptibly(barrier);
// Previous startListening() call should never have taken place and it should be ok to call it
// again.
surfaceTextureHelper.startListening(listener);
surfaceTextureHelper.dispose();
}
/**
* Test calling startListening() with a new listener after stopListening() has been called.
*/
@Test
@MediumTest
public void testRestartListeningWithNewListener() throws InterruptedException {
// Create SurfaceTextureHelper and listener.
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
surfaceTextureHelper.setTextureSize(/* textureWidth= */ 32, /* textureHeight= */ 32);
final MockTextureListener listener1 = new MockTextureListener();
surfaceTextureHelper.startListening(listener1);
// Create EglBase with the SurfaceTexture as target EGLSurface.
final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN);
eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
eglBase.makeCurrent();
// Assert no frame has been received yet.
listener1.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1);
// Draw and wait for one frame.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// swapBuffers() will ultimately trigger onTextureFrameAvailable().
eglBase.swapBuffers();
listener1.waitForTextureBuffer().release();
// Stop listening - |listener1| should not receive any textures after this.
surfaceTextureHelper.stopListening();
// Connect different listener.
final MockTextureListener listener2 = new MockTextureListener();
surfaceTextureHelper.startListening(listener2);
// Assert no frame has been received yet.
listener2.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1);
// Draw one frame.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
eglBase.swapBuffers();
// Check that |listener2| received the frame, and not |listener1|.
listener2.waitForTextureBuffer().release();
listener1.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1);
surfaceTextureHelper.dispose();
eglBase.release();
}
@Test
@MediumTest
public void testTexturetoYuv() throws InterruptedException {
final int width = 16;
final int height = 16;
final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN);
// Create SurfaceTextureHelper and listener.
final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create(
"SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext());
final MockTextureListener listener = new MockTextureListener();
surfaceTextureHelper.startListening(listener);
surfaceTextureHelper.setTextureSize(width, height);
// Create resources for stubbing an OES texture producer. |eglBase| has the SurfaceTexture in
// |surfaceTextureHelper| as the target EGLSurface.
eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
assertEquals(eglBase.surfaceWidth(), width);
assertEquals(eglBase.surfaceHeight(), height);
final int red[] = new int[] {79, 144, 185};
final int green[] = new int[] {66, 210, 162};
final int blue[] = new int[] {161, 117, 158};
final int ref_y[] = new int[] {85, 170, 161};
final int ref_u[] = new int[] {168, 97, 123};
final int ref_v[] = new int[] {127, 106, 138};
// Draw three frames.
for (int i = 0; i < 3; ++i) {
// Draw a constant color frame onto the SurfaceTexture.
eglBase.makeCurrent();
GLES20.glClearColor(red[i] / 255.0f, green[i] / 255.0f, blue[i] / 255.0f, 1.0f);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// swapBuffers() will ultimately trigger onTextureFrameAvailable().
eglBase.swapBuffers();
// Wait for an OES texture to arrive.
final VideoFrame.TextureBuffer textureBuffer = listener.waitForTextureBuffer();
final VideoFrame.I420Buffer i420 = textureBuffer.toI420();
textureBuffer.release();
// Memory layout: Lines are 16 bytes. First 16 lines are
// the Y data. These are followed by 8 lines with 8 bytes of U
// data on the left and 8 bytes of V data on the right.
//
// Offset
// 0 YYYYYYYY YYYYYYYY
// 16 YYYYYYYY YYYYYYYY
// ...
// 240 YYYYYYYY YYYYYYYY
// 256 UUUUUUUU VVVVVVVV
// 272 UUUUUUUU VVVVVVVV
// ...
// 368 UUUUUUUU VVVVVVVV
// 384 buffer end
// Allow off-by-one differences due to different rounding.
final ByteBuffer dataY = i420.getDataY();
final int strideY = i420.getStrideY();
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
assertClose(1, ref_y[i], dataY.get(y * strideY + x) & 0xFF);
}
}
final int chromaWidth = width / 2;
final int chromaHeight = height / 2;
final ByteBuffer dataU = i420.getDataU();
final ByteBuffer dataV = i420.getDataV();
final int strideU = i420.getStrideU();
final int strideV = i420.getStrideV();
for (int y = 0; y < chromaHeight; y++) {
for (int x = 0; x < chromaWidth; x++) {
assertClose(1, ref_u[i], dataU.get(y * strideU + x) & 0xFF);
assertClose(1, ref_v[i], dataV.get(y * strideV + x) & 0xFF);
}
}
i420.release();
}
surfaceTextureHelper.dispose();
eglBase.release();
}
}