webrtc/api/androidtests/src/org/webrtc/SurfaceTextureHelperTest.java

/*
 *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
package org.webrtc;

import android.graphics.SurfaceTexture;
import android.opengl.GLES20;
import android.os.SystemClock;
import android.test.ActivityTestCase;
import android.test.suitebuilder.annotation.MediumTest;
import android.test.suitebuilder.annotation.SmallTest;

import java.nio.ByteBuffer;
import java.util.concurrent.CountDownLatch;

public final class SurfaceTextureHelperTest extends ActivityTestCase {
  /**
   * Mock texture listener with blocking wait functionality.
   */
  public static final class MockTextureListener
      implements SurfaceTextureHelper.OnTextureFrameAvailableListener {
    public int oesTextureId;
    public float[] transformMatrix;
    private boolean hasNewFrame = false;
    // Thread where frames are expected to be received on.
    private final Thread expectedThread;

    MockTextureListener() {
      this.expectedThread = null;
    }

    MockTextureListener(Thread expectedThread) {
      this.expectedThread = expectedThread;
    }

    @Override
    public synchronized void onTextureFrameAvailable(
        int oesTextureId, float[] transformMatrix, long timestampNs) {
      if (expectedThread != null && Thread.currentThread() != expectedThread) {
        throw new IllegalStateException("onTextureFrameAvailable called on wrong thread.");
      }
      this.oesTextureId = oesTextureId;
      this.transformMatrix = transformMatrix;
      hasNewFrame = true;
      notifyAll();
    }

    /**
     * Wait indefinitely for a new frame.
     */
    public synchronized void waitForNewFrame() throws InterruptedException {
      while (!hasNewFrame) {
        wait();
      }
      hasNewFrame = false;
    }

    /**
     * Wait for a new frame, or until the specified timeout elapses. Returns true if a new frame was
     * received before the timeout.
     */
    public synchronized boolean waitForNewFrame(final long timeoutMs) throws InterruptedException {
      final long startTimeMs = SystemClock.elapsedRealtime();
      long timeRemainingMs = timeoutMs;
      while (!hasNewFrame && timeRemainingMs > 0) {
        wait(timeRemainingMs);
        final long elapsedTimeMs = SystemClock.elapsedRealtime() - startTimeMs;
        timeRemainingMs = timeoutMs - elapsedTimeMs;
      }
      final boolean didReceiveFrame = hasNewFrame;
      hasNewFrame = false;
      return didReceiveFrame;
    }
  }

  /** Assert that two integers are close, with difference at most
   * {@code threshold}. */
  public static void assertClose(int threshold, int expected, int actual) {
    if (Math.abs(expected - actual) <= threshold)
      return;
    failNotEquals("Not close enough, threshold " + threshold, expected, actual);
  }

  /**
   * Test normal use by receiving three uniform texture frames. Texture frames are returned as early
   * as possible. The texture pixel values are inspected by drawing the texture frame to a pixel
   * buffer and reading it back with glReadPixels().
   */
  @MediumTest
  public static void testThreeConstantColorFrames() throws InterruptedException {
    final int width = 16;
    final int height = 16;
    // Create EGL base with a pixel buffer as display output.
    final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PIXEL_BUFFER);
    eglBase.createPbufferSurface(width, height);
    final GlRectDrawer drawer = new GlRectDrawer();

    // Create SurfaceTextureHelper and listener.
    final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create(
        "SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext());
    final MockTextureListener listener = new MockTextureListener();
    surfaceTextureHelper.startListening(listener);
    surfaceTextureHelper.getSurfaceTexture().setDefaultBufferSize(width, height);

    // Create resources for stubbing an OES texture producer. |eglOesBase| has the SurfaceTexture in
    // |surfaceTextureHelper| as the target EGLSurface.
    final EglBase eglOesBase = EglBase.create(eglBase.getEglBaseContext(), EglBase.CONFIG_PLAIN);
    eglOesBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
    assertEquals(eglOesBase.surfaceWidth(), width);
    assertEquals(eglOesBase.surfaceHeight(), height);

    final int red[] = new int[] {79, 144, 185};
    final int green[] = new int[] {66, 210, 162};
    final int blue[] = new int[] {161, 117, 158};
    // Draw three frames.
    for (int i = 0; i < 3; ++i) {
      // Draw a constant color frame onto the SurfaceTexture.
      eglOesBase.makeCurrent();
      GLES20.glClearColor(red[i] / 255.0f, green[i] / 255.0f, blue[i] / 255.0f, 1.0f);
      GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
      // swapBuffers() will ultimately trigger onTextureFrameAvailable().
      eglOesBase.swapBuffers();

      // Wait for an OES texture to arrive and draw it onto the pixel buffer.
      listener.waitForNewFrame();
      eglBase.makeCurrent();
      drawer.drawOes(
          listener.oesTextureId, listener.transformMatrix, width, height, 0, 0, width, height);

      surfaceTextureHelper.returnTextureFrame();

      // Download the pixels in the pixel buffer as RGBA. Not all platforms support RGB, e.g.
      // Nexus 9.
      final ByteBuffer rgbaData = ByteBuffer.allocateDirect(width * height * 4);
      GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgbaData);
      GlUtil.checkNoGLES2Error("glReadPixels");

      // Assert rendered image is expected constant color.
      while (rgbaData.hasRemaining()) {
        assertEquals(rgbaData.get() & 0xFF, red[i]);
        assertEquals(rgbaData.get() & 0xFF, green[i]);
        assertEquals(rgbaData.get() & 0xFF, blue[i]);
        assertEquals(rgbaData.get() & 0xFF, 255);
      }
    }

    drawer.release();
    surfaceTextureHelper.dispose();
    eglBase.release();
  }

  /**
   * Test disposing the SurfaceTextureHelper while holding a pending texture frame. The pending
   * texture frame should still be valid, and this is tested by drawing the texture frame to a pixel
   * buffer and reading it back with glReadPixels().
   */
  @MediumTest
  public static void testLateReturnFrame() throws InterruptedException {
    final int width = 16;
    final int height = 16;
    // Create EGL base with a pixel buffer as display output.
    final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PIXEL_BUFFER);
    eglBase.createPbufferSurface(width, height);

    // Create SurfaceTextureHelper and listener.
    final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create(
        "SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext());
    final MockTextureListener listener = new MockTextureListener();
    surfaceTextureHelper.startListening(listener);
    surfaceTextureHelper.getSurfaceTexture().setDefaultBufferSize(width, height);

    // Create resources for stubbing an OES texture producer. |eglOesBase| has the SurfaceTexture in
    // |surfaceTextureHelper| as the target EGLSurface.
    final EglBase eglOesBase = EglBase.create(eglBase.getEglBaseContext(), EglBase.CONFIG_PLAIN);
    eglOesBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
    assertEquals(eglOesBase.surfaceWidth(), width);
    assertEquals(eglOesBase.surfaceHeight(), height);

    final int red = 79;
    final int green = 66;
    final int blue = 161;
    // Draw a constant color frame onto the SurfaceTexture.
    eglOesBase.makeCurrent();
    GLES20.glClearColor(red / 255.0f, green / 255.0f, blue / 255.0f, 1.0f);
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    // swapBuffers() will ultimately trigger onTextureFrameAvailable().
    eglOesBase.swapBuffers();
    eglOesBase.release();

    // Wait for OES texture frame.
    listener.waitForNewFrame();
    // Diconnect while holding the frame.
    surfaceTextureHelper.dispose();

    // Draw the pending texture frame onto the pixel buffer.
    eglBase.makeCurrent();
    final GlRectDrawer drawer = new GlRectDrawer();
    drawer.drawOes(
        listener.oesTextureId, listener.transformMatrix, width, height, 0, 0, width, height);
    drawer.release();

    // Download the pixels in the pixel buffer as RGBA. Not all platforms support RGB, e.g. Nexus 9.
    final ByteBuffer rgbaData = ByteBuffer.allocateDirect(width * height * 4);
    GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgbaData);
    GlUtil.checkNoGLES2Error("glReadPixels");
    eglBase.release();

    // Assert rendered image is expected constant color.
    while (rgbaData.hasRemaining()) {
      assertEquals(rgbaData.get() & 0xFF, red);
      assertEquals(rgbaData.get() & 0xFF, green);
      assertEquals(rgbaData.get() & 0xFF, blue);
      assertEquals(rgbaData.get() & 0xFF, 255);
    }
    // Late frame return after everything has been disposed and released.
    surfaceTextureHelper.returnTextureFrame();
  }

  /**
   * Test disposing the SurfaceTextureHelper, but keep trying to produce more texture frames. No
   * frames should be delivered to the listener.
   */
  @MediumTest
  public static void testDispose() throws InterruptedException {
    // Create SurfaceTextureHelper and listener.
    final SurfaceTextureHelper surfaceTextureHelper =
        SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
    final MockTextureListener listener = new MockTextureListener();
    surfaceTextureHelper.startListening(listener);
    // Create EglBase with the SurfaceTexture as target EGLSurface.
    final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN);
    eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
    eglBase.makeCurrent();
    // Assert no frame has been received yet.
    assertFalse(listener.waitForNewFrame(1));
    // Draw and wait for one frame.
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    // swapBuffers() will ultimately trigger onTextureFrameAvailable().
    eglBase.swapBuffers();
    listener.waitForNewFrame();
    surfaceTextureHelper.returnTextureFrame();

    // Dispose - we should not receive any textures after this.
    surfaceTextureHelper.dispose();

    // Draw one frame.
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    eglBase.swapBuffers();
    // swapBuffers() should not trigger onTextureFrameAvailable() because disposed has been called.
    // Assert that no OES texture was delivered.
    assertFalse(listener.waitForNewFrame(500));

    eglBase.release();
  }

  /**
   * Test disposing the SurfaceTextureHelper immediately after is has been setup to use a
   * shared context. No frames should be delivered to the listener.
   */
  @SmallTest
  public static void testDisposeImmediately() {
    final SurfaceTextureHelper surfaceTextureHelper =
        SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
    surfaceTextureHelper.dispose();
  }

  /**
   * Call stopListening(), but keep trying to produce more texture frames. No frames should be
   * delivered to the listener.
   */
  @MediumTest
  public static void testStopListening() throws InterruptedException {
    // Create SurfaceTextureHelper and listener.
    final SurfaceTextureHelper surfaceTextureHelper =
        SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
    final MockTextureListener listener = new MockTextureListener();
    surfaceTextureHelper.startListening(listener);
    // Create EglBase with the SurfaceTexture as target EGLSurface.
    final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN);
    eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
    eglBase.makeCurrent();
    // Assert no frame has been received yet.
    assertFalse(listener.waitForNewFrame(1));
    // Draw and wait for one frame.
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    // swapBuffers() will ultimately trigger onTextureFrameAvailable().
    eglBase.swapBuffers();
    listener.waitForNewFrame();
    surfaceTextureHelper.returnTextureFrame();

    // Stop listening - we should not receive any textures after this.
    surfaceTextureHelper.stopListening();

    // Draw one frame.
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    eglBase.swapBuffers();
    // swapBuffers() should not trigger onTextureFrameAvailable() because disposed has been called.
    // Assert that no OES texture was delivered.
    assertFalse(listener.waitForNewFrame(500));

    surfaceTextureHelper.dispose();
    eglBase.release();
  }

  /**
   * Test stopListening() immediately after the SurfaceTextureHelper has been setup.
   */
  @SmallTest
  public static void testStopListeningImmediately() throws InterruptedException {
    final SurfaceTextureHelper surfaceTextureHelper =
        SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
    final MockTextureListener listener = new MockTextureListener();
    surfaceTextureHelper.startListening(listener);
    surfaceTextureHelper.stopListening();
    surfaceTextureHelper.dispose();
  }

  /**
   * Test stopListening() immediately after the SurfaceTextureHelper has been setup on the handler
   * thread.
   */
  @SmallTest
  public static void testStopListeningImmediatelyOnHandlerThread() throws InterruptedException {
    final SurfaceTextureHelper surfaceTextureHelper =
        SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
    final MockTextureListener listener = new MockTextureListener();

    final CountDownLatch stopListeningBarrier = new CountDownLatch(1);
    final CountDownLatch stopListeningBarrierDone = new CountDownLatch(1);
    // Start by posting to the handler thread to keep it occupied.
    surfaceTextureHelper.getHandler().post(new Runnable() {
      @Override
      public void run() {
        ThreadUtils.awaitUninterruptibly(stopListeningBarrier);
        surfaceTextureHelper.stopListening();
        stopListeningBarrierDone.countDown();
      }
    });

    // startListening() is asynchronous and will post to the occupied handler thread.
    surfaceTextureHelper.startListening(listener);
    // Wait for stopListening() to be called on the handler thread.
    stopListeningBarrier.countDown();
    stopListeningBarrierDone.await();
    // Wait until handler thread is idle to try to catch late startListening() call.
    final CountDownLatch barrier = new CountDownLatch(1);
    surfaceTextureHelper.getHandler().post(new Runnable() {
      @Override
      public void run() {
        barrier.countDown();
      }
    });
    ThreadUtils.awaitUninterruptibly(barrier);
    // Previous startListening() call should never have taken place and it should be ok to call it
    // again.
    surfaceTextureHelper.startListening(listener);

    surfaceTextureHelper.dispose();
  }

  /**
   * Test calling startListening() with a new listener after stopListening() has been called.
   */
  @MediumTest
  public static void testRestartListeningWithNewListener() throws InterruptedException {
    // Create SurfaceTextureHelper and listener.
    final SurfaceTextureHelper surfaceTextureHelper =
        SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null);
    final MockTextureListener listener1 = new MockTextureListener();
    surfaceTextureHelper.startListening(listener1);
    // Create EglBase with the SurfaceTexture as target EGLSurface.
    final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN);
    eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
    eglBase.makeCurrent();
    // Assert no frame has been received yet.
    assertFalse(listener1.waitForNewFrame(1));
    // Draw and wait for one frame.
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    // swapBuffers() will ultimately trigger onTextureFrameAvailable().
    eglBase.swapBuffers();
    listener1.waitForNewFrame();
    surfaceTextureHelper.returnTextureFrame();

    // Stop listening - |listener1| should not receive any textures after this.
    surfaceTextureHelper.stopListening();

    // Connect different listener.
    final MockTextureListener listener2 = new MockTextureListener();
    surfaceTextureHelper.startListening(listener2);
    // Assert no frame has been received yet.
    assertFalse(listener2.waitForNewFrame(1));

    // Draw one frame.
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    eglBase.swapBuffers();

    // Check that |listener2| received the frame, and not |listener1|.
    listener2.waitForNewFrame();
    assertFalse(listener1.waitForNewFrame(1));

    surfaceTextureHelper.returnTextureFrame();

    surfaceTextureHelper.dispose();
    eglBase.release();
  }

  @MediumTest
  public static void testTexturetoYUV() throws InterruptedException {
    final int width = 16;
    final int height = 16;

    final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN);

    // Create SurfaceTextureHelper and listener.
    final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create(
        "SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext());
    final MockTextureListener listener = new MockTextureListener();
    surfaceTextureHelper.startListening(listener);
    surfaceTextureHelper.getSurfaceTexture().setDefaultBufferSize(width, height);

    // Create resources for stubbing an OES texture producer. |eglBase| has the SurfaceTexture in
    // |surfaceTextureHelper| as the target EGLSurface.

    eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
    assertEquals(eglBase.surfaceWidth(), width);
    assertEquals(eglBase.surfaceHeight(), height);

    final int red[] = new int[] {79, 144, 185};
    final int green[] = new int[] {66, 210, 162};
    final int blue[] = new int[] {161, 117, 158};

    final int ref_y[] = new int[] {81, 180, 168};
    final int ref_u[] = new int[] {173, 93, 122};
    final int ref_v[] = new int[] {127, 103, 140};

    // Draw three frames.
    for (int i = 0; i < 3; ++i) {
      // Draw a constant color frame onto the SurfaceTexture.
      eglBase.makeCurrent();
      GLES20.glClearColor(red[i] / 255.0f, green[i] / 255.0f, blue[i] / 255.0f, 1.0f);
      GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
      // swapBuffers() will ultimately trigger onTextureFrameAvailable().
      eglBase.swapBuffers();

      // Wait for an OES texture to arrive.
      listener.waitForNewFrame();

      // Memory layout: Lines are 16 bytes. First 16 lines are
      // the Y data. These are followed by 8 lines with 8 bytes of U
      // data on the left and 8 bytes of V data on the right.
      //
      // Offset
      //      0 YYYYYYYY YYYYYYYY
      //     16 YYYYYYYY YYYYYYYY
      //    ...
      //    240 YYYYYYYY YYYYYYYY
      //    256 UUUUUUUU VVVVVVVV
      //    272 UUUUUUUU VVVVVVVV
      //    ...
      //    368 UUUUUUUU VVVVVVVV
      //    384 buffer end
      ByteBuffer buffer = ByteBuffer.allocateDirect(width * height * 3 / 2);
      surfaceTextureHelper.textureToYUV(
          buffer, width, height, width, listener.oesTextureId, listener.transformMatrix);

      surfaceTextureHelper.returnTextureFrame();

      // Allow off-by-one differences due to different rounding.
      while (buffer.position() < width * height) {
        assertClose(1, buffer.get() & 0xff, ref_y[i]);
      }
      while (buffer.hasRemaining()) {
        if (buffer.position() % width < width / 2)
          assertClose(1, buffer.get() & 0xff, ref_u[i]);
        else
          assertClose(1, buffer.get() & 0xff, ref_v[i]);
      }
    }

    surfaceTextureHelper.dispose();
    eglBase.release();
  }
}