| /* |
| * Copyright 2015 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "sdk/android/src/jni/videoframe.h" |
| |
| #include <memory> |
| |
| #include "common_video/include/video_frame_buffer.h" |
| #include "rtc_base/bind.h" |
| #include "rtc_base/checks.h" |
| #include "rtc_base/keep_ref_until_done.h" |
| #include "rtc_base/logging.h" |
| #include "rtc_base/memory/aligned_malloc.h" |
| #include "rtc_base/scoped_ref_ptr.h" |
| #include "rtc_base/timeutils.h" |
| #include "sdk/android/generated_video_jni/jni/VideoFrame_jni.h" |
| #include "sdk/android/src/jni/jni_helpers.h" |
| #include "sdk/android/src/jni/wrapped_native_i420_buffer.h" |
| #include "third_party/libyuv/include/libyuv/scale.h" |
| |
| namespace webrtc { |
| namespace jni { |
| |
| namespace { |
| |
| class AndroidVideoI420Buffer : public I420BufferInterface { |
| public: |
| // Adopts and takes ownership of the Java VideoFrame.Buffer. I.e. retain() |
| // will not be called, but release() will be called when the returned |
| // AndroidVideoBuffer is destroyed. |
| static rtc::scoped_refptr<AndroidVideoI420Buffer> Adopt( |
| JNIEnv* jni, |
| int width, |
| int height, |
| const JavaRef<jobject>& j_video_frame_buffer); |
| |
| protected: |
| // Should not be called directly. Adopts the buffer. Use Adopt() instead for |
| // clarity. |
| AndroidVideoI420Buffer(JNIEnv* jni, |
| int width, |
| int height, |
| const JavaRef<jobject>& j_video_frame_buffer); |
| ~AndroidVideoI420Buffer() override; |
| |
| private: |
| const uint8_t* DataY() const override { return data_y_; } |
| const uint8_t* DataU() const override { return data_u_; } |
| const uint8_t* DataV() const override { return data_v_; } |
| |
| int StrideY() const override { return stride_y_; } |
| int StrideU() const override { return stride_u_; } |
| int StrideV() const override { return stride_v_; } |
| |
| int width() const override { return width_; } |
| int height() const override { return height_; } |
| |
| const int width_; |
| const int height_; |
| // Holds a VideoFrame.I420Buffer. |
| const ScopedJavaGlobalRef<jobject> j_video_frame_buffer_; |
| |
| const uint8_t* data_y_; |
| const uint8_t* data_u_; |
| const uint8_t* data_v_; |
| int stride_y_; |
| int stride_u_; |
| int stride_v_; |
| }; |
| |
| rtc::scoped_refptr<AndroidVideoI420Buffer> AndroidVideoI420Buffer::Adopt( |
| JNIEnv* jni, |
| int width, |
| int height, |
| const JavaRef<jobject>& j_video_frame_buffer) { |
| return new rtc::RefCountedObject<AndroidVideoI420Buffer>( |
| jni, width, height, j_video_frame_buffer); |
| } |
| |
| AndroidVideoI420Buffer::AndroidVideoI420Buffer( |
| JNIEnv* jni, |
| int width, |
| int height, |
| const JavaRef<jobject>& j_video_frame_buffer) |
| : width_(width), |
| height_(height), |
| j_video_frame_buffer_(jni, j_video_frame_buffer) { |
| ScopedJavaLocalRef<jobject> j_data_y = |
| Java_I420Buffer_getDataY(jni, j_video_frame_buffer); |
| ScopedJavaLocalRef<jobject> j_data_u = |
| Java_I420Buffer_getDataU(jni, j_video_frame_buffer); |
| ScopedJavaLocalRef<jobject> j_data_v = |
| Java_I420Buffer_getDataV(jni, j_video_frame_buffer); |
| |
| data_y_ = |
| static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_data_y.obj())); |
| data_u_ = |
| static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_data_u.obj())); |
| data_v_ = |
| static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_data_v.obj())); |
| |
| stride_y_ = Java_I420Buffer_getStrideY(jni, j_video_frame_buffer); |
| stride_u_ = Java_I420Buffer_getStrideU(jni, j_video_frame_buffer); |
| stride_v_ = Java_I420Buffer_getStrideV(jni, j_video_frame_buffer); |
| } |
| |
| AndroidVideoI420Buffer::~AndroidVideoI420Buffer() { |
| JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| Java_Buffer_release(jni, j_video_frame_buffer_); |
| } |
| |
| } // namespace |
| |
| int64_t GetJavaVideoFrameTimestampNs(JNIEnv* jni, |
| const JavaRef<jobject>& j_video_frame) { |
| return Java_VideoFrame_getTimestampNs(jni, j_video_frame); |
| } |
| |
| Matrix::Matrix(JNIEnv* jni, const JavaRef<jfloatArray>& a) { |
| RTC_CHECK_EQ(16, jni->GetArrayLength(a.obj())); |
| jni->GetFloatArrayRegion(a.obj(), 0, 16, elem_); |
| } |
| |
| ScopedJavaLocalRef<jfloatArray> Matrix::ToJava(JNIEnv* jni) const { |
| ScopedJavaLocalRef<jfloatArray> matrix(jni, jni->NewFloatArray(16)); |
| jni->SetFloatArrayRegion(matrix.obj(), 0, 16, elem_); |
| return matrix; |
| } |
| |
| void Matrix::Rotate(VideoRotation rotation) { |
| // Texture coordinates are in the range 0 to 1. The transformation of the last |
| // row in each rotation matrix is needed for proper translation, e.g, to |
| // mirror x, we don't replace x by -x, but by 1-x. |
| switch (rotation) { |
| case kVideoRotation_0: |
| break; |
| case kVideoRotation_90: { |
| const float ROTATE_90[16] = {elem_[4], |
| elem_[5], |
| elem_[6], |
| elem_[7], |
| -elem_[0], |
| -elem_[1], |
| -elem_[2], |
| -elem_[3], |
| elem_[8], |
| elem_[9], |
| elem_[10], |
| elem_[11], |
| elem_[0] + elem_[12], |
| elem_[1] + elem_[13], |
| elem_[2] + elem_[14], |
| elem_[3] + elem_[15]}; |
| memcpy(elem_, ROTATE_90, sizeof(elem_)); |
| } break; |
| case kVideoRotation_180: { |
| const float ROTATE_180[16] = {-elem_[0], |
| -elem_[1], |
| -elem_[2], |
| -elem_[3], |
| -elem_[4], |
| -elem_[5], |
| -elem_[6], |
| -elem_[7], |
| elem_[8], |
| elem_[9], |
| elem_[10], |
| elem_[11], |
| elem_[0] + elem_[4] + elem_[12], |
| elem_[1] + elem_[5] + elem_[13], |
| elem_[2] + elem_[6] + elem_[14], |
| elem_[3] + elem_[11] + elem_[15]}; |
| memcpy(elem_, ROTATE_180, sizeof(elem_)); |
| } break; |
| case kVideoRotation_270: { |
| const float ROTATE_270[16] = {-elem_[4], |
| -elem_[5], |
| -elem_[6], |
| -elem_[7], |
| elem_[0], |
| elem_[1], |
| elem_[2], |
| elem_[3], |
| elem_[8], |
| elem_[9], |
| elem_[10], |
| elem_[11], |
| elem_[4] + elem_[12], |
| elem_[5] + elem_[13], |
| elem_[6] + elem_[14], |
| elem_[7] + elem_[15]}; |
| memcpy(elem_, ROTATE_270, sizeof(elem_)); |
| } break; |
| } |
| } |
| |
| // Calculates result = a * b, in column-major order. |
| void Matrix::Multiply(const float a[16], const float b[16], float result[16]) { |
| for (int i = 0; i < 4; ++i) { |
| for (int j = 0; j < 4; ++j) { |
| float sum = 0; |
| for (int k = 0; k < 4; ++k) { |
| sum += a[k * 4 + j] * b[i * 4 + k]; |
| } |
| result[i * 4 + j] = sum; |
| } |
| } |
| } |
| |
| // Center crop by keeping xFraction of the width and yFraction of the height, |
| // so e.g. cropping from 640x480 to 640x360 would use |
| // xFraction=1, yFraction=360/480. |
| void Matrix::Crop(float xFraction, |
| float yFraction, |
| float xOffset, |
| float yOffset) { |
| const float crop_matrix[16] = {xFraction, 0, 0, 0, 0, yFraction, 0, 0, |
| 0, 0, 1, 0, xOffset, yOffset, 0, 1}; |
| const Matrix old = *this; |
| Multiply(crop_matrix, old.elem_, this->elem_); |
| } |
| |
| NativeHandleImpl::NativeHandleImpl(int id, const Matrix& matrix) |
| : oes_texture_id(id), sampling_matrix(matrix) {} |
| |
| NativeHandleImpl::NativeHandleImpl( |
| JNIEnv* jni, |
| jint j_oes_texture_id, |
| const JavaRef<jfloatArray>& j_transform_matrix) |
| : oes_texture_id(j_oes_texture_id), |
| sampling_matrix(jni, j_transform_matrix) {} |
| |
| rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBuffer::Adopt( |
| JNIEnv* jni, |
| const JavaRef<jobject>& j_video_frame_buffer) { |
| return new rtc::RefCountedObject<AndroidVideoBuffer>(jni, |
| j_video_frame_buffer); |
| } |
| |
| rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBuffer::Create( |
| JNIEnv* jni, |
| const JavaRef<jobject>& j_video_frame_buffer) { |
| Java_Buffer_retain(jni, j_video_frame_buffer); |
| return Adopt(jni, j_video_frame_buffer); |
| } |
| |
| AndroidVideoBuffer::AndroidVideoBuffer( |
| JNIEnv* jni, |
| const JavaRef<jobject>& j_video_frame_buffer) |
| : width_(Java_Buffer_getWidth(jni, j_video_frame_buffer)), |
| height_(Java_Buffer_getHeight(jni, j_video_frame_buffer)), |
| j_video_frame_buffer_(jni, j_video_frame_buffer) {} |
| |
| AndroidVideoBuffer::~AndroidVideoBuffer() { |
| JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| Java_Buffer_release(jni, j_video_frame_buffer_); |
| } |
| |
| const ScopedJavaGlobalRef<jobject>& AndroidVideoBuffer::video_frame_buffer() |
| const { |
| return j_video_frame_buffer_; |
| } |
| |
| rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBuffer::CropAndScale( |
| JNIEnv* jni, |
| int crop_x, |
| int crop_y, |
| int crop_width, |
| int crop_height, |
| int scale_width, |
| int scale_height) { |
| return Adopt(jni, Java_Buffer_cropAndScale(jni, j_video_frame_buffer_, crop_x, |
| crop_y, crop_width, crop_height, |
| scale_width, scale_height)); |
| } |
| |
| VideoFrameBuffer::Type AndroidVideoBuffer::type() const { |
| return Type::kNative; |
| } |
| |
| int AndroidVideoBuffer::width() const { |
| return width_; |
| } |
| |
| int AndroidVideoBuffer::height() const { |
| return height_; |
| } |
| |
| rtc::scoped_refptr<I420BufferInterface> AndroidVideoBuffer::ToI420() { |
| JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| ScopedJavaLocalRef<jobject> j_i420_buffer = |
| Java_Buffer_toI420(jni, j_video_frame_buffer_); |
| |
| // We don't need to retain the buffer because toI420 returns a new object that |
| // we are assumed to take the ownership of. |
| return AndroidVideoI420Buffer::Adopt(jni, width_, height_, j_i420_buffer); |
| } |
| |
| VideoFrame JavaToNativeFrame(JNIEnv* jni, |
| const JavaRef<jobject>& j_video_frame, |
| uint32_t timestamp_rtp) { |
| ScopedJavaLocalRef<jobject> j_video_frame_buffer = |
| Java_VideoFrame_getBuffer(jni, j_video_frame); |
| int rotation = Java_VideoFrame_getRotation(jni, j_video_frame); |
| int64_t timestamp_ns = Java_VideoFrame_getTimestampNs(jni, j_video_frame); |
| rtc::scoped_refptr<AndroidVideoBuffer> buffer = |
| AndroidVideoBuffer::Create(jni, j_video_frame_buffer); |
| return VideoFrame(buffer, timestamp_rtp, |
| timestamp_ns / rtc::kNumNanosecsPerMillisec, |
| static_cast<VideoRotation>(rotation)); |
| } |
| |
| ScopedJavaLocalRef<jobject> NativeToJavaVideoFrame(JNIEnv* jni, |
| const VideoFrame& frame) { |
| rtc::scoped_refptr<VideoFrameBuffer> buffer = frame.video_frame_buffer(); |
| |
| if (buffer->type() == VideoFrameBuffer::Type::kNative) { |
| AndroidVideoBuffer* android_buffer = |
| static_cast<AndroidVideoBuffer*>(buffer.get()); |
| ScopedJavaLocalRef<jobject> j_video_frame_buffer( |
| jni, android_buffer->video_frame_buffer()); |
| Java_Buffer_retain(jni, j_video_frame_buffer); |
| return Java_VideoFrame_Constructor( |
| jni, j_video_frame_buffer, static_cast<jint>(frame.rotation()), |
| static_cast<jlong>(frame.timestamp_us() * |
| rtc::kNumNanosecsPerMicrosec)); |
| } else { |
| return Java_VideoFrame_Constructor( |
| jni, WrapI420Buffer(jni, buffer->ToI420()), |
| static_cast<jint>(frame.rotation()), |
| static_cast<jlong>(frame.timestamp_us() * |
| rtc::kNumNanosecsPerMicrosec)); |
| } |
| } |
| |
| void ReleaseJavaVideoFrame(JNIEnv* jni, const JavaRef<jobject>& j_video_frame) { |
| Java_VideoFrame_release(jni, j_video_frame); |
| } |
| |
| static void JNI_VideoFrame_CropAndScaleI420( |
| JNIEnv* jni, |
| const JavaParamRef<jclass>&, |
| const JavaParamRef<jobject>& j_src_y, |
| jint src_stride_y, |
| const JavaParamRef<jobject>& j_src_u, |
| jint src_stride_u, |
| const JavaParamRef<jobject>& j_src_v, |
| jint src_stride_v, |
| jint crop_x, |
| jint crop_y, |
| jint crop_width, |
| jint crop_height, |
| const JavaParamRef<jobject>& j_dst_y, |
| jint dst_stride_y, |
| const JavaParamRef<jobject>& j_dst_u, |
| jint dst_stride_u, |
| const JavaParamRef<jobject>& j_dst_v, |
| jint dst_stride_v, |
| jint scale_width, |
| jint scale_height) { |
| uint8_t const* src_y = |
| static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_src_y.obj())); |
| uint8_t const* src_u = |
| static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_src_u.obj())); |
| uint8_t const* src_v = |
| static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_src_v.obj())); |
| uint8_t* dst_y = |
| static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_y.obj())); |
| uint8_t* dst_u = |
| static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_u.obj())); |
| uint8_t* dst_v = |
| static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_v.obj())); |
| |
| // Perform cropping using pointer arithmetic. |
| src_y += crop_x + crop_y * src_stride_y; |
| src_u += crop_x / 2 + crop_y / 2 * src_stride_u; |
| src_v += crop_x / 2 + crop_y / 2 * src_stride_v; |
| |
| bool ret = libyuv::I420Scale( |
| src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, crop_width, |
| crop_height, dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v, |
| dst_stride_v, scale_width, scale_height, libyuv::kFilterBox); |
| RTC_DCHECK_EQ(ret, 0) << "I420Scale failed"; |
| } |
| |
| } // namespace jni |
| } // namespace webrtc |
