Android: Update MediaCodecVideoDecoder to output VideoFrames
Bug: webrtc:9181
Change-Id: I7eba15167536e453956c511a056143b039f52b92
Reviewed-on: https://webrtc-review.googlesource.com/71664
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22988}diff --git a/sdk/android/api/org/webrtc/MediaCodecVideoDecoder.java b/sdk/android/api/org/webrtc/MediaCodecVideoDecoder.java
index 8ab033d..a9368db 100644
--- a/sdk/android/api/org/webrtc/MediaCodecVideoDecoder.java
+++ b/sdk/android/api/org/webrtc/MediaCodecVideoDecoder.java
@@ -30,6 +30,8 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
+import org.webrtc.EglBase;
+import org.webrtc.VideoFrame;
// Java-side of peerconnection.cc:MediaCodecVideoDecoder.
// This class is an implementation detail of the Java PeerConnection API.
@@ -320,16 +322,16 @@
}
}
- // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer output.
+ // Pass null in |eglContext| to configure the codec for ByteBuffer output.
@CalledByNativeUnchecked
- private boolean initDecode(VideoCodecType type, int width, int height,
- @Nullable SurfaceTextureHelper surfaceTextureHelper) {
+ private boolean initDecode(
+ VideoCodecType type, int width, int height, @Nullable EglBase.Context eglContext) {
if (mediaCodecThread != null) {
throw new RuntimeException("initDecode: Forgot to release()?");
}
String mime = null;
- useSurface = (surfaceTextureHelper != null);
+ useSurface = (eglContext != null);
String[] supportedCodecPrefixes = null;
if (type == VideoCodecType.VIDEO_CODEC_VP8) {
mime = VP8_MIME_TYPE;
@@ -359,9 +361,14 @@
stride = width;
sliceHeight = height;
- if (useSurface && surfaceTextureHelper != null) {
- textureListener = new TextureListener(surfaceTextureHelper);
- surface = new Surface(surfaceTextureHelper.getSurfaceTexture());
+ if (useSurface) {
+ @Nullable
+ final SurfaceTextureHelper surfaceTextureHelper =
+ SurfaceTextureHelper.create("Decoder SurfaceTextureHelper", eglContext);
+ if (surfaceTextureHelper != null) {
+ textureListener = new TextureListener(surfaceTextureHelper);
+ surface = new Surface(surfaceTextureHelper.getSurfaceTexture());
+ }
}
MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
@@ -567,8 +574,7 @@
// Helper struct for dequeueTextureBuffer() below.
private static class DecodedTextureBuffer {
- private final int textureID;
- private final float[] transformMatrix;
+ private final VideoFrame.Buffer videoFrameBuffer;
// Presentation timestamp returned in dequeueOutputBuffer call.
private final long presentationTimeStampMs;
// C++ inputImage._timeStamp value for output frame.
@@ -585,11 +591,9 @@
// A DecodedTextureBuffer with zero |textureID| has special meaning and represents a frame
// that was dropped.
- public DecodedTextureBuffer(int textureID, float[] transformMatrix,
- long presentationTimeStampMs, long timeStampMs, long ntpTimeStampMs, long decodeTimeMs,
- long frameDelay) {
- this.textureID = textureID;
- this.transformMatrix = transformMatrix;
+ public DecodedTextureBuffer(VideoFrame.Buffer videoFrameBuffer, long presentationTimeStampMs,
+ long timeStampMs, long ntpTimeStampMs, long decodeTimeMs, long frameDelay) {
+ this.videoFrameBuffer = videoFrameBuffer;
this.presentationTimeStampMs = presentationTimeStampMs;
this.timeStampMs = timeStampMs;
this.ntpTimeStampMs = ntpTimeStampMs;
@@ -598,13 +602,8 @@
}
@CalledByNative("DecodedTextureBuffer")
- int getTextureId() {
- return textureID;
- }
-
- @CalledByNative("DecodedTextureBuffer")
- float[] getTransformMatrix() {
- return transformMatrix;
+ VideoFrame.Buffer getVideoFrameBuffer() {
+ return videoFrameBuffer;
}
@CalledByNative("DecodedTextureBuffer")
@@ -634,8 +633,7 @@
}
// Poll based texture listener.
- private static class TextureListener
- implements SurfaceTextureHelper.OnTextureFrameAvailableListener {
+ private class TextureListener implements SurfaceTextureHelper.OnTextureFrameAvailableListener {
private final SurfaceTextureHelper surfaceTextureHelper;
// |newFrameLock| is used to synchronize arrival of new frames with wait()/notifyAll().
private final Object newFrameLock = new Object();
@@ -674,9 +672,10 @@
throw new IllegalStateException("Already holding a texture.");
}
// |timestampNs| is always zero on some Android versions.
- renderedBuffer = new DecodedTextureBuffer(oesTextureId, transformMatrix,
- bufferToRender.presentationTimeStampMs, bufferToRender.timeStampMs,
- bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs,
+ final VideoFrame.Buffer buffer = surfaceTextureHelper.createTextureBuffer(
+ width, height, RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix));
+ renderedBuffer = new DecodedTextureBuffer(buffer, bufferToRender.presentationTimeStampMs,
+ bufferToRender.timeStampMs, bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs,
SystemClock.elapsedRealtime() - bufferToRender.endDecodeTimeMs);
bufferToRender = null;
newFrameLock.notifyAll();
@@ -709,10 +708,11 @@
surfaceTextureHelper.stopListening();
synchronized (newFrameLock) {
if (renderedBuffer != null) {
- surfaceTextureHelper.returnTextureFrame();
+ renderedBuffer.getVideoFrameBuffer().release();
renderedBuffer = null;
}
}
+ surfaceTextureHelper.dispose();
}
}
@@ -844,8 +844,9 @@
}
mediaCodec.releaseOutputBuffer(droppedFrame.index, false /* render */);
- return new DecodedTextureBuffer(0, null, droppedFrame.presentationTimeStampMs,
- droppedFrame.timeStampMs, droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs,
+ return new DecodedTextureBuffer(null /* videoFrameBuffer */,
+ droppedFrame.presentationTimeStampMs, droppedFrame.timeStampMs,
+ droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs,
SystemClock.elapsedRealtime() - droppedFrame.endDecodeTimeMs);
}
return null;
diff --git a/sdk/android/src/jni/androidmediadecoder.cc b/sdk/android/src/jni/androidmediadecoder.cc
index 0a3d81a7..88cfb96 100644
--- a/sdk/android/src/jni/androidmediadecoder.cc
+++ b/sdk/android/src/jni/androidmediadecoder.cc
@@ -112,7 +112,6 @@
bool use_surface_;
VideoCodec codec_;
I420BufferPool decoded_frame_pool_;
- rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_;
DecodedImageCallback* callback_;
int frames_received_; // Number of frames received by decoder.
int frames_decoded_; // Number of frames decoded by decoder.
@@ -224,26 +223,12 @@
ResetVariables();
- if (use_surface_) {
- surface_texture_helper_ = SurfaceTextureHelper::create(
- jni, "Decoder SurfaceTextureHelper",
- JavaParamRef<jobject>(render_egl_context_));
- if (!surface_texture_helper_) {
- ALOGE << "Couldn't create SurfaceTextureHelper - fallback to SW codec";
- sw_fallback_required_ = true;
- return WEBRTC_VIDEO_CODEC_ERROR;
- }
- }
-
ScopedJavaLocalRef<jobject> j_video_codec_enum =
Java_VideoCodecType_fromNativeIndex(jni, codecType_);
- jobject j_surface_texture_helper =
- use_surface_
- ? surface_texture_helper_->GetJavaSurfaceTextureHelper().obj()
- : nullptr;
+ jobject j_egl_context = use_surface_ ? render_egl_context_ : nullptr;
bool success = Java_MediaCodecVideoDecoder_initDecode(
jni, j_media_codec_video_decoder_, j_video_codec_enum, codec_.width,
- codec_.height, JavaParamRef<jobject>(j_surface_texture_helper));
+ codec_.height, JavaParamRef<jobject>(j_egl_context));
if (CheckException(jni) || !success) {
ALOGE << "Codec initialization error - fallback to SW codec.";
@@ -325,7 +310,6 @@
ScopedLocalRefFrame local_ref_frame(jni);
input_buffers_.clear();
Java_MediaCodecVideoDecoder_release(jni, j_media_codec_video_decoder_);
- surface_texture_helper_ = nullptr;
inited_ = false;
rtc::MessageQueueManager::Clear(this);
if (CheckException(jni)) {
@@ -604,18 +588,14 @@
decode_time_ms =
Java_DecodedTextureBuffer_getDecodeTimeMs(jni, j_decoder_output_buffer);
- const int texture_id =
- Java_DecodedTextureBuffer_getTextureId(jni, j_decoder_output_buffer);
- if (texture_id != 0) { // |texture_id| == 0 represents a dropped frame.
- ScopedJavaLocalRef<jfloatArray> j_transform_matrix =
- Java_DecodedTextureBuffer_getTransformMatrix(jni,
- j_decoder_output_buffer);
+ ScopedJavaLocalRef<jobject> j_video_frame_buffer =
+ Java_DecodedTextureBuffer_getVideoFrameBuffer(jni,
+ j_decoder_output_buffer);
+ // |video_frame_buffer| == null represents a dropped frame.
+ if (!j_video_frame_buffer.is_null()) {
frame_delayed_ms = Java_DecodedTextureBuffer_getFrameDelayMs(
jni, j_decoder_output_buffer);
-
- // Create VideoFrameBuffer with native texture handle.
- frame_buffer = surface_texture_helper_->CreateTextureFrame(
- width, height, NativeHandleImpl(jni, texture_id, j_transform_matrix));
+ frame_buffer = AndroidVideoBuffer::Adopt(jni, j_video_frame_buffer);
} else {
EnableFrameLogOnWarning();
}
