blob: 1b9b56608d9bda9598deda7c5702bf5fc2d3f272 [file] [log] [blame]
/*
* Copyright 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.robolectric.Shadows.shadowOf;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaFormat;
import android.os.Handler;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.chromium.testing.local.LocalRobolectricTestRunner;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.robolectric.annotation.Config;
import org.webrtc.EncodedImage.FrameType;
import org.webrtc.FakeMediaCodecWrapper.State;
import org.webrtc.VideoDecoder.DecodeInfo;
import org.webrtc.VideoFrame.I420Buffer;
import org.webrtc.VideoFrame.TextureBuffer.Type;
@RunWith(LocalRobolectricTestRunner.class)
@Config(manifest = Config.NONE)
public class AndroidVideoDecoderTest {
private static final VideoDecoder.Settings TEST_DECODER_SETTINGS =
new VideoDecoder.Settings(/* numberOfCores= */ 1, /* width= */ 640, /* height= */ 480);
private static final int COLOR_FORMAT = CodecCapabilities.COLOR_FormatYUV420Planar;
private static final long POLL_DELAY_MS = 10;
private static final long DELIVER_DECODED_IMAGE_DELAY_MS = 10;
private static final byte[] ENCODED_TEST_DATA = new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
private class TestDecoder extends AndroidVideoDecoder {
private final Object deliverDecodedFrameLock = new Object();
private boolean deliverDecodedFrameDone = true;
public TestDecoder(MediaCodecWrapperFactory mediaCodecFactory, String codecName,
VideoCodecType codecType, int colorFormat, EglBase.Context sharedContext) {
super(mediaCodecFactory, codecName, codecType, colorFormat, sharedContext);
}
public void waitDeliverDecodedFrame() throws InterruptedException {
synchronized (deliverDecodedFrameLock) {
deliverDecodedFrameDone = false;
deliverDecodedFrameLock.notifyAll();
while (!deliverDecodedFrameDone) {
deliverDecodedFrameLock.wait();
}
}
}
@SuppressWarnings("WaitNotInLoop") // This method is called inside a loop.
@Override
protected void deliverDecodedFrame() {
synchronized (deliverDecodedFrameLock) {
if (deliverDecodedFrameDone) {
try {
deliverDecodedFrameLock.wait(DELIVER_DECODED_IMAGE_DELAY_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
}
}
if (deliverDecodedFrameDone) {
return;
}
super.deliverDecodedFrame();
deliverDecodedFrameDone = true;
deliverDecodedFrameLock.notifyAll();
}
}
@Override
protected SurfaceTextureHelper createSurfaceTextureHelper() {
return mockSurfaceTextureHelper;
}
@Override
protected void releaseSurface() {}
@Override
protected VideoFrame.I420Buffer allocateI420Buffer(int width, int height) {
int chromaHeight = (height + 1) / 2;
int strideUV = (width + 1) / 2;
int yPos = 0;
int uPos = yPos + width * height;
int vPos = uPos + strideUV * chromaHeight;
ByteBuffer buffer = ByteBuffer.allocateDirect(width * height + 2 * strideUV * chromaHeight);
buffer.position(yPos);
buffer.limit(uPos);
ByteBuffer dataY = buffer.slice();
buffer.position(uPos);
buffer.limit(vPos);
ByteBuffer dataU = buffer.slice();
buffer.position(vPos);
buffer.limit(vPos + strideUV * chromaHeight);
ByteBuffer dataV = buffer.slice();
return JavaI420Buffer.wrap(width, height, dataY, width, dataU, strideUV, dataV, strideUV,
/* releaseCallback= */ null);
}
@Override
protected void copyPlane(
ByteBuffer src, int srcStride, ByteBuffer dst, int dstStride, int width, int height) {
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
dst.put(y * dstStride + x, src.get(y * srcStride + x));
}
}
}
}
private class TestDecoderBuilder {
private VideoCodecType codecType = VideoCodecType.VP8;
private boolean useSurface = true;
public TestDecoderBuilder setCodecType(VideoCodecType codecType) {
this.codecType = codecType;
return this;
}
public TestDecoderBuilder setUseSurface(boolean useSurface) {
this.useSurface = useSurface;
return this;
}
public TestDecoder build() {
return new TestDecoder((String name)
-> fakeMediaCodecWrapper,
/* codecName= */ "org.webrtc.testdecoder", codecType, COLOR_FORMAT,
useSurface ? mockEglBaseContext : null);
}
}
private static class FakeDecoderCallback implements VideoDecoder.Callback {
public final List<VideoFrame> decodedFrames;
public FakeDecoderCallback() {
decodedFrames = new ArrayList<>();
}
@Override
public void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp) {
frame.retain();
decodedFrames.add(frame);
}
public void release() {
for (VideoFrame frame : decodedFrames) frame.release();
decodedFrames.clear();
}
}
private EncodedImage createTestEncodedImage() {
return EncodedImage.builder()
.setBuffer(ByteBuffer.wrap(ENCODED_TEST_DATA))
.setFrameType(FrameType.VideoFrameKey)
.setCompleteFrame(true)
.createEncodedImage();
}
@Mock private EglBase.Context mockEglBaseContext;
@Mock private SurfaceTextureHelper mockSurfaceTextureHelper;
@Mock private VideoDecoder.Callback mockDecoderCallback;
private FakeMediaCodecWrapper fakeMediaCodecWrapper;
private FakeDecoderCallback fakeDecoderCallback;
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
when(mockSurfaceTextureHelper.getSurfaceTexture())
.thenReturn(new SurfaceTexture(/*texName=*/0));
MediaFormat outputFormat = new MediaFormat();
// TODO(sakal): Add more details to output format as needed.
fakeMediaCodecWrapper = spy(new FakeMediaCodecWrapper(outputFormat));
fakeDecoderCallback = new FakeDecoderCallback();
}
@After
public void cleanUp() {
fakeDecoderCallback.release();
}
@Test
public void testInit() {
// Set-up.
AndroidVideoDecoder decoder = new TestDecoderBuilder().setCodecType(VideoCodecType.VP8).build();
// Test.
assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback))
.isEqualTo(VideoCodecStatus.OK);
// Verify.
assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.EXECUTING_RUNNING);
MediaFormat mediaFormat = fakeMediaCodecWrapper.getConfiguredFormat();
assertThat(mediaFormat).isNotNull();
assertThat(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
.isEqualTo(TEST_DECODER_SETTINGS.width);
assertThat(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT))
.isEqualTo(TEST_DECODER_SETTINGS.height);
assertThat(mediaFormat.getString(MediaFormat.KEY_MIME))
.isEqualTo(VideoCodecType.VP8.mimeType());
}
@Test
public void testRelease() {
// Set-up.
AndroidVideoDecoder decoder = new TestDecoderBuilder().build();
decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback);
// Test.
assertThat(decoder.release()).isEqualTo(VideoCodecStatus.OK);
// Verify.
assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.RELEASED);
}
@Test
public void testReleaseMultipleTimes() {
// Set-up.
AndroidVideoDecoder decoder = new TestDecoderBuilder().build();
decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback);
// Test.
assertThat(decoder.release()).isEqualTo(VideoCodecStatus.OK);
assertThat(decoder.release()).isEqualTo(VideoCodecStatus.OK);
// Verify.
assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.RELEASED);
}
@Test
public void testDecodeQueuesData() {
// Set-up.
AndroidVideoDecoder decoder = new TestDecoderBuilder().build();
decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback);
// Test.
assertThat(decoder.decode(createTestEncodedImage(),
new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0)))
.isEqualTo(VideoCodecStatus.OK);
// Verify.
ArgumentCaptor<Integer> indexCaptor = ArgumentCaptor.forClass(Integer.class);
ArgumentCaptor<Integer> offsetCaptor = ArgumentCaptor.forClass(Integer.class);
ArgumentCaptor<Integer> sizeCaptor = ArgumentCaptor.forClass(Integer.class);
verify(fakeMediaCodecWrapper)
.queueInputBuffer(indexCaptor.capture(), offsetCaptor.capture(), sizeCaptor.capture(),
/* presentationTimeUs= */ anyLong(),
/* flags= */ eq(0));
ByteBuffer inputBuffer = fakeMediaCodecWrapper.getInputBuffers()[indexCaptor.getValue()];
CodecTestHelper.assertEqualContents(
ENCODED_TEST_DATA, inputBuffer, offsetCaptor.getValue(), sizeCaptor.getValue());
}
@Test
public void testDeliversOutputByteBuffers() throws InterruptedException {
final byte[] testOutputData = CodecTestHelper.generateRandomData(
TEST_DECODER_SETTINGS.width * TEST_DECODER_SETTINGS.height * 3 / 2);
final I420Buffer expectedDeliveredBuffer = CodecTestHelper.wrapI420(
TEST_DECODER_SETTINGS.width, TEST_DECODER_SETTINGS.height, testOutputData);
// Set-up.
TestDecoder decoder = new TestDecoderBuilder().setUseSurface(/* useSurface = */ false).build();
decoder.initDecode(TEST_DECODER_SETTINGS, fakeDecoderCallback);
decoder.decode(createTestEncodedImage(),
new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0));
fakeMediaCodecWrapper.addOutputData(
testOutputData, /* presentationTimestampUs= */ 0, /* flags= */ 0);
// Test.
decoder.waitDeliverDecodedFrame();
// Verify.
assertThat(fakeDecoderCallback.decodedFrames).hasSize(1);
VideoFrame videoFrame = fakeDecoderCallback.decodedFrames.get(0);
assertThat(videoFrame).isNotNull();
assertThat(videoFrame.getRotatedWidth()).isEqualTo(TEST_DECODER_SETTINGS.width);
assertThat(videoFrame.getRotatedHeight()).isEqualTo(TEST_DECODER_SETTINGS.height);
assertThat(videoFrame.getRotation()).isEqualTo(0);
I420Buffer deliveredBuffer = videoFrame.getBuffer().toI420();
assertThat(deliveredBuffer.getDataY()).isEqualTo(expectedDeliveredBuffer.getDataY());
assertThat(deliveredBuffer.getDataU()).isEqualTo(expectedDeliveredBuffer.getDataU());
assertThat(deliveredBuffer.getDataV()).isEqualTo(expectedDeliveredBuffer.getDataV());
}
@Test
public void testRendersOutputTexture() throws InterruptedException {
// Set-up.
TestDecoder decoder = new TestDecoderBuilder().build();
decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback);
decoder.decode(createTestEncodedImage(),
new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0));
int bufferIndex =
fakeMediaCodecWrapper.addOutputTexture(/* presentationTimestampUs= */ 0, /* flags= */ 0);
// Test.
decoder.waitDeliverDecodedFrame();
// Verify.
verify(fakeMediaCodecWrapper).releaseOutputBuffer(bufferIndex, /* render= */ true);
}
@Test
public void testSurfaceTextureStall_FramesDropped() throws InterruptedException {
final int numFrames = 10;
// Maximum number of frame the decoder can keep queued on the output side.
final int maxQueuedBuffers = 3;
// Set-up.
TestDecoder decoder = new TestDecoderBuilder().build();
decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback);
// Test.
int[] bufferIndices = new int[numFrames];
for (int i = 0; i < 10; i++) {
decoder.decode(createTestEncodedImage(),
new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0));
bufferIndices[i] =
fakeMediaCodecWrapper.addOutputTexture(/* presentationTimestampUs= */ 0, /* flags= */ 0);
decoder.waitDeliverDecodedFrame();
}
// Verify.
InOrder releaseOrder = inOrder(fakeMediaCodecWrapper);
releaseOrder.verify(fakeMediaCodecWrapper)
.releaseOutputBuffer(bufferIndices[0], /* render= */ true);
for (int i = 1; i < numFrames - maxQueuedBuffers; i++) {
releaseOrder.verify(fakeMediaCodecWrapper)
.releaseOutputBuffer(bufferIndices[i], /* render= */ false);
}
}
@Test
public void testDeliversRenderedBuffers() throws InterruptedException {
// Set-up.
TestDecoder decoder = new TestDecoderBuilder().build();
decoder.initDecode(TEST_DECODER_SETTINGS, fakeDecoderCallback);
decoder.decode(createTestEncodedImage(),
new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0));
fakeMediaCodecWrapper.addOutputTexture(/* presentationTimestampUs= */ 0, /* flags= */ 0);
// Render the output buffer.
decoder.waitDeliverDecodedFrame();
ArgumentCaptor<VideoSink> videoSinkCaptor = ArgumentCaptor.forClass(VideoSink.class);
verify(mockSurfaceTextureHelper).startListening(videoSinkCaptor.capture());
// Test.
Runnable releaseCallback = mock(Runnable.class);
VideoFrame.TextureBuffer outputTextureBuffer =
new TextureBufferImpl(TEST_DECODER_SETTINGS.width, TEST_DECODER_SETTINGS.height, Type.OES,
/* id= */ 0,
/* transformMatrix= */ new Matrix(),
/* toI420Handler= */ new Handler(), new YuvConverter(), releaseCallback);
VideoFrame outputVideoFrame =
new VideoFrame(outputTextureBuffer, /* rotation= */ 0, /* timestampNs= */ 0);
videoSinkCaptor.getValue().onFrame(outputVideoFrame);
outputVideoFrame.release();
// Verify.
assertThat(fakeDecoderCallback.decodedFrames).hasSize(1);
VideoFrame videoFrame = fakeDecoderCallback.decodedFrames.get(0);
assertThat(videoFrame).isNotNull();
assertThat(videoFrame.getBuffer()).isEqualTo(outputTextureBuffer);
fakeDecoderCallback.release();
verify(releaseCallback).run();
}
@Test
public void testConfigureExceptionTriggerSWFallback() {
// Set-up.
doThrow(new IllegalStateException("Fake error"))
.when(fakeMediaCodecWrapper)
.configure(any(), any(), any(), anyInt());
AndroidVideoDecoder decoder = new TestDecoderBuilder().build();
// Test.
assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback))
.isEqualTo(VideoCodecStatus.FALLBACK_SOFTWARE);
}
@Test
public void testStartExceptionTriggerSWFallback() {
// Set-up.
doThrow(new IllegalStateException("Fake error")).when(fakeMediaCodecWrapper).start();
AndroidVideoDecoder decoder = new TestDecoderBuilder().build();
// Test.
assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback))
.isEqualTo(VideoCodecStatus.FALLBACK_SOFTWARE);
}
}