| /* | |
| * Copyright 2016 The WebRTC Project Authors. All rights reserved. | |
| * | |
| * Use of this source code is governed by a BSD-style license | |
| * that can be found in the LICENSE file in the root of the source | |
| * tree. An additional intellectual property rights grant can be found | |
| * in the file PATENTS. All contributing project authors may | |
| * be found in the AUTHORS file in the root of the source tree. | |
| */ | |
| package org.webrtc; | |
| import static org.junit.Assert.assertEquals; | |
| import static org.junit.Assert.assertFalse; | |
| import static org.junit.Assert.assertNotNull; | |
| import static org.junit.Assert.assertNull; | |
| import static org.junit.Assert.assertTrue; | |
| import static org.junit.Assert.fail; | |
| import android.graphics.Bitmap; | |
| import android.graphics.SurfaceTexture; | |
| import android.opengl.GLES11Ext; | |
| import android.opengl.GLES20; | |
| import androidx.test.InstrumentationRegistry; | |
| import androidx.test.filters.SmallTest; | |
| import java.nio.ByteBuffer; | |
| import java.util.ArrayList; | |
| import java.util.Arrays; | |
| import java.util.Collection; | |
| import java.util.concurrent.CountDownLatch; | |
| import org.junit.After; | |
| import org.junit.Before; | |
| import org.junit.Test; | |
| import org.junit.runner.RunWith; | |
| import org.junit.runners.Parameterized; | |
| import org.junit.runners.Parameterized.Parameter; | |
| import org.junit.runners.Parameterized.Parameters; | |
| // EmptyActivity is needed for the surface. | |
| @RunWith(Parameterized.class) | |
| public class EglRendererTest { | |
| private final static String TAG = "EglRendererTest"; | |
| private final static int RENDER_WAIT_MS = 1000; | |
| private final static int SURFACE_WAIT_MS = 1000; | |
| private final static int TEST_FRAME_WIDTH = 4; | |
| private final static int TEST_FRAME_HEIGHT = 4; | |
| private final static int REMOVE_FRAME_LISTENER_RACY_NUM_TESTS = 10; | |
| // Some arbitrary frames. | |
| private final static byte[][][] TEST_FRAMES_DATA = { | |
| { | |
| new byte[] { | |
| -99, -93, -88, -83, -78, -73, -68, -62, -56, -52, -46, -41, -36, -31, -26, -20}, | |
| new byte[] {110, 113, 116, 118}, new byte[] {31, 45, 59, 73}, | |
| }, | |
| { | |
| new byte[] { | |
| -108, -103, -98, -93, -87, -82, -77, -72, -67, -62, -56, -50, -45, -40, -35, -30}, | |
| new byte[] {120, 123, 125, -127}, new byte[] {87, 100, 114, 127}, | |
| }, | |
| { | |
| new byte[] { | |
| -117, -112, -107, -102, -97, -92, -87, -81, -75, -71, -65, -60, -55, -50, -44, -39}, | |
| new byte[] {113, 116, 118, 120}, new byte[] {45, 59, 73, 87}, | |
| }, | |
| }; | |
| private final static ByteBuffer[][] TEST_FRAMES = | |
| copyTestDataToDirectByteBuffers(TEST_FRAMES_DATA); | |
| private static class TestFrameListener implements EglRenderer.FrameListener { | |
| final private ArrayList<Bitmap> bitmaps = new ArrayList<Bitmap>(); | |
| boolean bitmapReceived; | |
| Bitmap storedBitmap; | |
| @Override | |
| // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. | |
| @SuppressWarnings("NoSynchronizedMethodCheck") | |
| public synchronized void onFrame(Bitmap bitmap) { | |
| if (bitmapReceived) { | |
| fail("Unexpected bitmap was received."); | |
| } | |
| bitmapReceived = true; | |
| storedBitmap = bitmap; | |
| notify(); | |
| } | |
| // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. | |
| @SuppressWarnings("NoSynchronizedMethodCheck") | |
| public synchronized boolean waitForBitmap(int timeoutMs) throws InterruptedException { | |
| final long endTimeMs = System.currentTimeMillis() + timeoutMs; | |
| while (!bitmapReceived) { | |
| final long waitTimeMs = endTimeMs - System.currentTimeMillis(); | |
| if (waitTimeMs < 0) { | |
| return false; | |
| } | |
| wait(timeoutMs); | |
| } | |
| return true; | |
| } | |
| // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. | |
| @SuppressWarnings("NoSynchronizedMethodCheck") | |
| public synchronized Bitmap resetAndGetBitmap() { | |
| bitmapReceived = false; | |
| return storedBitmap; | |
| } | |
| } | |
| @Parameter(0) | |
| public boolean useRenderSynchronizer; | |
| final TestFrameListener testFrameListener = new TestFrameListener(); | |
| EglRenderer eglRenderer; | |
| EglThread eglThread; | |
| CountDownLatch surfaceReadyLatch = new CountDownLatch(1); | |
| int oesTextureId; | |
| SurfaceTexture surfaceTexture; | |
| @Parameters | |
| public static Collection<Object[]> data() { | |
| return Arrays.asList(new Object[] {true}, new Object[] {false}); | |
| } | |
| @Before | |
| public void setUp() throws Exception { | |
| PeerConnectionFactory.initialize(PeerConnectionFactory.InitializationOptions | |
| .builder(InstrumentationRegistry.getTargetContext()) | |
| .setNativeLibraryName(TestConstants.NATIVE_LIBRARY) | |
| .createInitializationOptions()); | |
| RenderSynchronizer renderSynchronizer = useRenderSynchronizer ? new RenderSynchronizer() : null; | |
| eglThread = | |
| EglThread.create( | |
| null /* releaseMonitor */, | |
| null /* sharedContext */, | |
| EglBase.CONFIG_RGBA, | |
| renderSynchronizer); | |
| eglRenderer = new EglRenderer("TestRenderer: "); | |
| eglRenderer.init(eglThread, new GlRectDrawer(), false /* usePresentationTimeStamp */); | |
| oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | |
| surfaceTexture = new SurfaceTexture(oesTextureId); | |
| surfaceTexture.setDefaultBufferSize(1 /* width */, 1 /* height */); | |
| eglRenderer.createEglSurface(surfaceTexture); | |
| } | |
| @After | |
| public void tearDown() { | |
| surfaceTexture.release(); | |
| GLES20.glDeleteTextures(1 /* n */, new int[] {oesTextureId}, 0 /* offset */); | |
| eglRenderer.release(); | |
| } | |
| /** Checks the bitmap is not null and the correct size. */ | |
| private static void checkBitmap(Bitmap bitmap, float scale) { | |
| assertNotNull(bitmap); | |
| assertEquals((int) (TEST_FRAME_WIDTH * scale), bitmap.getWidth()); | |
| assertEquals((int) (TEST_FRAME_HEIGHT * scale), bitmap.getHeight()); | |
| } | |
| /** | |
| * Does linear sampling on U/V plane of test data. | |
| * | |
| * @param data Plane data to be sampled from. | |
| * @param planeWidth Width of the plane data. This is also assumed to be the stride. | |
| * @param planeHeight Height of the plane data. | |
| * @param x X-coordinate in range [0, 1]. | |
| * @param y Y-coordinate in range [0, 1]. | |
| */ | |
| private static float linearSample( | |
| ByteBuffer plane, int planeWidth, int planeHeight, float x, float y) { | |
| final int stride = planeWidth; | |
| final float coordX = x * planeWidth; | |
| final float coordY = y * planeHeight; | |
| int lowIndexX = (int) Math.floor(coordX - 0.5f); | |
| int lowIndexY = (int) Math.floor(coordY - 0.5f); | |
| int highIndexX = lowIndexX + 1; | |
| int highIndexY = lowIndexY + 1; | |
| final float highWeightX = coordX - lowIndexX - 0.5f; | |
| final float highWeightY = coordY - lowIndexY - 0.5f; | |
| final float lowWeightX = 1f - highWeightX; | |
| final float lowWeightY = 1f - highWeightY; | |
| // Clamp on the edges. | |
| lowIndexX = Math.max(0, lowIndexX); | |
| lowIndexY = Math.max(0, lowIndexY); | |
| highIndexX = Math.min(planeWidth - 1, highIndexX); | |
| highIndexY = Math.min(planeHeight - 1, highIndexY); | |
| float lowYValue = (plane.get(lowIndexY * stride + lowIndexX) & 0xFF) * lowWeightX | |
| + (plane.get(lowIndexY * stride + highIndexX) & 0xFF) * highWeightX; | |
| float highYValue = (plane.get(highIndexY * stride + lowIndexX) & 0xFF) * lowWeightX | |
| + (plane.get(highIndexY * stride + highIndexX) & 0xFF) * highWeightX; | |
| return lowWeightY * lowYValue + highWeightY * highYValue; | |
| } | |
| private static byte saturatedFloatToByte(float c) { | |
| return (byte) Math.round(255f * Math.max(0f, Math.min(1f, c))); | |
| } | |
| /** | |
| * Converts test data YUV frame to expected RGBA frame. Tries to match the behavior of OpenGL | |
| * YUV drawer shader. Does linear sampling on the U- and V-planes. | |
| * | |
| * @param yuvFrame Array of size 3 containing Y-, U-, V-planes for image of size | |
| * (TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT). U- and V-planes should be half the size | |
| * of the Y-plane. | |
| */ | |
| private static byte[] convertYUVFrameToRGBA(ByteBuffer[] yuvFrame) { | |
| final byte[] argbFrame = new byte[TEST_FRAME_WIDTH * TEST_FRAME_HEIGHT * 4]; | |
| final int argbStride = TEST_FRAME_WIDTH * 4; | |
| final int yStride = TEST_FRAME_WIDTH; | |
| final int vStride = TEST_FRAME_WIDTH / 2; | |
| for (int y = 0; y < TEST_FRAME_HEIGHT; y++) { | |
| for (int x = 0; x < TEST_FRAME_WIDTH; x++) { | |
| final float yC = ((yuvFrame[0].get(y * yStride + x) & 0xFF) - 16f) / 219f; | |
| final float uC = (linearSample(yuvFrame[1], TEST_FRAME_WIDTH / 2, TEST_FRAME_HEIGHT / 2, | |
| (x + 0.5f) / TEST_FRAME_WIDTH, (y + 0.5f) / TEST_FRAME_HEIGHT) | |
| - 16f) | |
| / 224f | |
| - 0.5f; | |
| final float vC = (linearSample(yuvFrame[2], TEST_FRAME_WIDTH / 2, TEST_FRAME_HEIGHT / 2, | |
| (x + 0.5f) / TEST_FRAME_WIDTH, (y + 0.5f) / TEST_FRAME_HEIGHT) | |
| - 16f) | |
| / 224f | |
| - 0.5f; | |
| final float rC = yC + 1.403f * vC; | |
| final float gC = yC - 0.344f * uC - 0.714f * vC; | |
| final float bC = yC + 1.77f * uC; | |
| argbFrame[y * argbStride + x * 4 + 0] = saturatedFloatToByte(rC); | |
| argbFrame[y * argbStride + x * 4 + 1] = saturatedFloatToByte(gC); | |
| argbFrame[y * argbStride + x * 4 + 2] = saturatedFloatToByte(bC); | |
| argbFrame[y * argbStride + x * 4 + 3] = (byte) 255; | |
| } | |
| } | |
| return argbFrame; | |
| } | |
| /** Checks that the bitmap content matches the test frame with the given index. */ | |
| // TODO(titovartem) make correct fix during webrtc:9175 | |
| @SuppressWarnings("ByteBufferBackingArray") | |
| private static void checkBitmapContent(Bitmap bitmap, int frame) { | |
| checkBitmap(bitmap, 1f); | |
| byte[] expectedRGBA = convertYUVFrameToRGBA(TEST_FRAMES[frame]); | |
| ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(bitmap.getByteCount()); | |
| bitmap.copyPixelsToBuffer(bitmapBuffer); | |
| for (int i = 0; i < expectedRGBA.length; i++) { | |
| int expected = expectedRGBA[i] & 0xFF; | |
| int value = bitmapBuffer.get(i) & 0xFF; | |
| // Due to unknown conversion differences check value matches +-1. | |
| if (Math.abs(value - expected) > 1) { | |
| Logging.d(TAG, "Expected bitmap content: " + Arrays.toString(expectedRGBA)); | |
| Logging.d(TAG, "Bitmap content: " + Arrays.toString(bitmapBuffer.array())); | |
| fail("Frame doesn't match original frame on byte " + i + ". Expected: " + expected | |
| + " Result: " + value); | |
| } | |
| } | |
| } | |
| private void drainRenderThread() throws Exception { | |
| CountDownLatch latch = new CountDownLatch(1); | |
| eglThread.getHandler().post(() -> latch.countDown()); | |
| latch.await(); | |
| } | |
| /** Tells eglRenderer to render test frame with given index. */ | |
| private void feedFrame(int i) { | |
| final VideoFrame.I420Buffer buffer = JavaI420Buffer.wrap(TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT, | |
| TEST_FRAMES[i][0], TEST_FRAME_WIDTH, TEST_FRAMES[i][1], TEST_FRAME_WIDTH / 2, | |
| TEST_FRAMES[i][2], TEST_FRAME_WIDTH / 2, null /* releaseCallback */); | |
| final VideoFrame frame = new VideoFrame(buffer, 0 /* rotation */, 0 /* timestamp */); | |
| eglRenderer.onFrame(frame); | |
| frame.release(); | |
| } | |
| @Test | |
| @SmallTest | |
| public void testAddFrameListener() throws Exception { | |
| eglRenderer.addFrameListener(testFrameListener, 0f /* scaleFactor */); | |
| feedFrame(0); | |
| assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| assertNull(testFrameListener.resetAndGetBitmap()); | |
| eglRenderer.addFrameListener(testFrameListener, 0f /* scaleFactor */); | |
| feedFrame(1); | |
| assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| assertNull(testFrameListener.resetAndGetBitmap()); | |
| feedFrame(2); | |
| // Check we get no more bitmaps than two. | |
| assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| } | |
| @Test | |
| @SmallTest | |
| public void testAddFrameListenerBitmap() throws Exception { | |
| eglRenderer.addFrameListener(testFrameListener, 1f /* scaleFactor */); | |
| feedFrame(0); | |
| assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| checkBitmapContent(testFrameListener.resetAndGetBitmap(), 0); | |
| eglRenderer.addFrameListener(testFrameListener, 1f /* scaleFactor */); | |
| feedFrame(1); | |
| assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| checkBitmapContent(testFrameListener.resetAndGetBitmap(), 1); | |
| } | |
| @Test | |
| @SmallTest | |
| public void testAddFrameListenerBitmapScale() throws Exception { | |
| for (int i = 0; i < 3; ++i) { | |
| float scale = i * 0.5f + 0.5f; | |
| eglRenderer.addFrameListener(testFrameListener, scale); | |
| feedFrame(i); | |
| assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| checkBitmap(testFrameListener.resetAndGetBitmap(), scale); | |
| } | |
| } | |
| /** | |
| * Checks that the frame listener will not be called with a frame that was delivered before the | |
| * frame listener was added. | |
| */ | |
| @Test | |
| @SmallTest | |
| public void testFrameListenerNotCalledWithOldFrames() throws Exception { | |
| feedFrame(0); | |
| eglRenderer.addFrameListener(testFrameListener, 0f); | |
| // Check the old frame does not trigger frame listener. | |
| assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| } | |
| /** Checks that the frame listener will not be called after it is removed. */ | |
| @Test | |
| @SmallTest | |
| public void testRemoveFrameListenerNotRacy() throws Exception { | |
| for (int i = 0; i < REMOVE_FRAME_LISTENER_RACY_NUM_TESTS; i++) { | |
| feedFrame(0); | |
| eglRenderer.addFrameListener(testFrameListener, 0f); | |
| eglRenderer.removeFrameListener(testFrameListener); | |
| feedFrame(1); | |
| } | |
| // Check the frame listener hasn't triggered. | |
| assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| } | |
| @Test | |
| @SmallTest | |
| public void testFrameListenersFpsReduction() throws Exception { | |
| // Test that normal frame listeners receive frames while the renderer is paused. | |
| eglRenderer.pauseVideo(); | |
| eglRenderer.addFrameListener(testFrameListener, 1f /* scaleFactor */); | |
| feedFrame(0); | |
| assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| checkBitmapContent(testFrameListener.resetAndGetBitmap(), 0); | |
| // Test that frame listeners with FPS reduction applied receive frames while the renderer is not | |
| // paused. | |
| eglRenderer.disableFpsReduction(); | |
| eglRenderer.addFrameListener( | |
| testFrameListener, 1f /* scaleFactor */, null, true /* applyFpsReduction */); | |
| feedFrame(1); | |
| assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| checkBitmapContent(testFrameListener.resetAndGetBitmap(), 1); | |
| // Test that frame listeners with FPS reduction applied will not receive frames while the | |
| // renderer is paused. | |
| eglRenderer.pauseVideo(); | |
| eglRenderer.addFrameListener( | |
| testFrameListener, 1f /* scaleFactor */, null, true /* applyFpsReduction */); | |
| feedFrame(1); | |
| assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); | |
| } | |
| @Test | |
| @SmallTest | |
| public void testRecreateSurface() throws Exception { | |
| // Make sure the EGLSurface has been created. | |
| drainRenderThread(); | |
| // Relase the surface and wait for completion. | |
| CountDownLatch latch = new CountDownLatch(1); | |
| eglRenderer.releaseEglSurface(() -> latch.countDown()); | |
| latch.await(); | |
| // Recreate the surface. | |
| eglRenderer.createEglSurface(surfaceTexture); | |
| drainRenderThread(); | |
| } | |
| private static ByteBuffer[][] copyTestDataToDirectByteBuffers(byte[][][] testData) { | |
| final ByteBuffer[][] result = new ByteBuffer[testData.length][]; | |
| for (int i = 0; i < testData.length; i++) { | |
| result[i] = new ByteBuffer[testData[i].length]; | |
| for (int j = 0; j < testData[i].length; j++) { | |
| result[i][j] = ByteBuffer.allocateDirect(testData[i][j].length); | |
| result[i][j].put(testData[i][j]); | |
| result[i][j].rewind(); | |
| } | |
| } | |
| return result; | |
| } | |
| } |