blob: ea1e04ed11fe6f662f6b4c1c48982f97cf236967 [file] [log] [blame]
/*
* Copyright 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "sdk/android/src/jni/native_handle_impl.h"
#include <memory>
#include "common_video/include/video_frame_buffer.h"
#include "rtc_base/bind.h"
#include "rtc_base/checks.h"
#include "rtc_base/keep_ref_until_done.h"
#include "rtc_base/logging.h"
#include "rtc_base/scoped_ref_ptr.h"
#include "rtc_base/timeutils.h"
#include "sdk/android/src/jni/classreferenceholder.h"
#include "sdk/android/src/jni/jni_helpers.h"
#include "sdk/android/src/jni/wrapped_native_i420_buffer.h"
#include "system_wrappers/include/aligned_malloc.h"
namespace webrtc {
namespace jni {
namespace {
class AndroidVideoI420Buffer : public I420BufferInterface {
public:
// Wraps an existing reference to a Java VideoBuffer. Retain will not be
// called but release will be called when the C++ object is destroyed.
static rtc::scoped_refptr<AndroidVideoI420Buffer> WrapReference(
JNIEnv* jni,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer);
protected:
AndroidVideoI420Buffer(JNIEnv* jni,
jmethodID j_retain_id,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer);
// Should not be called directly. Wraps a reference. Use
// AndroidVideoI420Buffer::WrapReference instead for clarity.
AndroidVideoI420Buffer(JNIEnv* jni,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer);
~AndroidVideoI420Buffer();
private:
const uint8_t* DataY() const override { return data_y_; }
const uint8_t* DataU() const override { return data_u_; }
const uint8_t* DataV() const override { return data_v_; }
int StrideY() const override { return stride_y_; }
int StrideU() const override { return stride_u_; }
int StrideV() const override { return stride_v_; }
int width() const override { return width_; }
int height() const override { return height_; }
const jmethodID j_release_id_;
const int width_;
const int height_;
// Holds a VideoFrame.I420Buffer.
const ScopedGlobalRef<jobject> j_video_frame_buffer_;
const uint8_t* data_y_;
const uint8_t* data_u_;
const uint8_t* data_v_;
int stride_y_;
int stride_u_;
int stride_v_;
};
rtc::scoped_refptr<AndroidVideoI420Buffer>
AndroidVideoI420Buffer::WrapReference(JNIEnv* jni,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer) {
return new rtc::RefCountedObject<AndroidVideoI420Buffer>(
jni, j_release_id, width, height, j_video_frame_buffer);
}
AndroidVideoI420Buffer::AndroidVideoI420Buffer(JNIEnv* jni,
jmethodID j_retain_id,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer)
: AndroidVideoI420Buffer(jni,
j_release_id,
width,
height,
j_video_frame_buffer) {
jni->CallVoidMethod(j_video_frame_buffer, j_retain_id);
}
AndroidVideoI420Buffer::AndroidVideoI420Buffer(JNIEnv* jni,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer)
: j_release_id_(j_release_id),
width_(width),
height_(height),
j_video_frame_buffer_(jni, j_video_frame_buffer) {
jclass j_video_frame_i420_buffer_class =
FindClass(jni, "org/webrtc/VideoFrame$I420Buffer");
jmethodID j_get_data_y_id = jni->GetMethodID(
j_video_frame_i420_buffer_class, "getDataY", "()Ljava/nio/ByteBuffer;");
jmethodID j_get_data_u_id = jni->GetMethodID(
j_video_frame_i420_buffer_class, "getDataU", "()Ljava/nio/ByteBuffer;");
jmethodID j_get_data_v_id = jni->GetMethodID(
j_video_frame_i420_buffer_class, "getDataV", "()Ljava/nio/ByteBuffer;");
jmethodID j_get_stride_y_id =
jni->GetMethodID(j_video_frame_i420_buffer_class, "getStrideY", "()I");
jmethodID j_get_stride_u_id =
jni->GetMethodID(j_video_frame_i420_buffer_class, "getStrideU", "()I");
jmethodID j_get_stride_v_id =
jni->GetMethodID(j_video_frame_i420_buffer_class, "getStrideV", "()I");
jobject j_data_y =
jni->CallObjectMethod(j_video_frame_buffer, j_get_data_y_id);
jobject j_data_u =
jni->CallObjectMethod(j_video_frame_buffer, j_get_data_u_id);
jobject j_data_v =
jni->CallObjectMethod(j_video_frame_buffer, j_get_data_v_id);
data_y_ = static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_data_y));
data_u_ = static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_data_u));
data_v_ = static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_data_v));
stride_y_ = jni->CallIntMethod(j_video_frame_buffer, j_get_stride_y_id);
stride_u_ = jni->CallIntMethod(j_video_frame_buffer, j_get_stride_u_id);
stride_v_ = jni->CallIntMethod(j_video_frame_buffer, j_get_stride_v_id);
}
AndroidVideoI420Buffer::~AndroidVideoI420Buffer() {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
jni->CallVoidMethod(*j_video_frame_buffer_, j_release_id_);
}
} // namespace
Matrix::Matrix(JNIEnv* jni, jfloatArray a) {
RTC_CHECK_EQ(16, jni->GetArrayLength(a));
jfloat* ptr = jni->GetFloatArrayElements(a, nullptr);
for (int i = 0; i < 16; ++i) {
elem_[i] = ptr[i];
}
jni->ReleaseFloatArrayElements(a, ptr, 0);
}
jfloatArray Matrix::ToJava(JNIEnv* jni) const {
jfloatArray matrix = jni->NewFloatArray(16);
jni->SetFloatArrayRegion(matrix, 0, 16, elem_);
return matrix;
}
void Matrix::Rotate(VideoRotation rotation) {
// Texture coordinates are in the range 0 to 1. The transformation of the last
// row in each rotation matrix is needed for proper translation, e.g, to
// mirror x, we don't replace x by -x, but by 1-x.
switch (rotation) {
case kVideoRotation_0:
break;
case kVideoRotation_90: {
const float ROTATE_90[16] =
{ elem_[4], elem_[5], elem_[6], elem_[7],
-elem_[0], -elem_[1], -elem_[2], -elem_[3],
elem_[8], elem_[9], elem_[10], elem_[11],
elem_[0] + elem_[12], elem_[1] + elem_[13],
elem_[2] + elem_[14], elem_[3] + elem_[15]};
memcpy(elem_, ROTATE_90, sizeof(elem_));
} break;
case kVideoRotation_180: {
const float ROTATE_180[16] =
{ -elem_[0], -elem_[1], -elem_[2], -elem_[3],
-elem_[4], -elem_[5], -elem_[6], -elem_[7],
elem_[8], elem_[9], elem_[10], elem_[11],
elem_[0] + elem_[4] + elem_[12], elem_[1] + elem_[5] + elem_[13],
elem_[2] + elem_[6] + elem_[14], elem_[3] + elem_[11]+ elem_[15]};
memcpy(elem_, ROTATE_180, sizeof(elem_));
} break;
case kVideoRotation_270: {
const float ROTATE_270[16] =
{ -elem_[4], -elem_[5], -elem_[6], -elem_[7],
elem_[0], elem_[1], elem_[2], elem_[3],
elem_[8], elem_[9], elem_[10], elem_[11],
elem_[4] + elem_[12], elem_[5] + elem_[13],
elem_[6] + elem_[14], elem_[7] + elem_[15]};
memcpy(elem_, ROTATE_270, sizeof(elem_));
} break;
}
}
// Calculates result = a * b, in column-major order.
void Matrix::Multiply(const float a[16], const float b[16], float result[16]) {
for (int i = 0; i < 4; ++i) {
for (int j = 0; j < 4; ++j) {
float sum = 0;
for (int k = 0; k < 4; ++k) {
sum += a[k * 4 + j] * b[i * 4 + k];
}
result[i * 4 + j] = sum;
}
}
}
// Center crop by keeping xFraction of the width and yFraction of the height,
// so e.g. cropping from 640x480 to 640x360 would use
// xFraction=1, yFraction=360/480.
void Matrix::Crop(float xFraction,
float yFraction,
float xOffset,
float yOffset) {
const float crop_matrix[16] =
{xFraction, 0, 0, 0,
0, yFraction, 0, 0,
0, 0, 1, 0,
xOffset, yOffset, 0, 1};
const Matrix old = *this;
Multiply(crop_matrix, old.elem_, this->elem_);
}
// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD.
static const int kBufferAlignment = 64;
NativeHandleImpl::NativeHandleImpl(int id, const Matrix& matrix)
: oes_texture_id(id), sampling_matrix(matrix) {}
NativeHandleImpl::NativeHandleImpl(JNIEnv* jni,
jint j_oes_texture_id,
jfloatArray j_transform_matrix)
: oes_texture_id(j_oes_texture_id),
sampling_matrix(jni, j_transform_matrix) {}
AndroidTextureBuffer::AndroidTextureBuffer(
int width,
int height,
const NativeHandleImpl& native_handle,
jobject surface_texture_helper,
const rtc::Callback0<void>& no_longer_used)
: width_(width),
height_(height),
native_handle_(native_handle),
surface_texture_helper_(surface_texture_helper),
no_longer_used_cb_(no_longer_used) {}
AndroidTextureBuffer::~AndroidTextureBuffer() {
no_longer_used_cb_();
}
VideoFrameBuffer::Type AndroidTextureBuffer::type() const {
return Type::kNative;
}
NativeHandleImpl AndroidTextureBuffer::native_handle_impl() const {
return native_handle_;
}
int AndroidTextureBuffer::width() const {
return width_;
}
int AndroidTextureBuffer::height() const {
return height_;
}
rtc::scoped_refptr<I420BufferInterface> AndroidTextureBuffer::ToI420() {
int uv_width = (width() + 7) / 8;
int stride = 8 * uv_width;
int uv_height = (height() + 1) / 2;
size_t size = stride * (height() + uv_height);
// The data is owned by the frame, and the normal case is that the
// data is deleted by the frame's destructor callback.
//
// TODO(nisse): Use an I420BufferPool. We then need to extend that
// class, and I420Buffer, to support our memory layout.
// TODO(nisse): Depending on
// system_wrappers/include/aligned_malloc.h violate current DEPS
// rules. We get away for now only because it is indirectly included
// by i420_buffer.h
std::unique_ptr<uint8_t, AlignedFreeDeleter> yuv_data(
static_cast<uint8_t*>(AlignedMalloc(size, kBufferAlignment)));
// See YuvConverter.java for the required layout.
uint8_t* y_data = yuv_data.get();
uint8_t* u_data = y_data + height() * stride;
uint8_t* v_data = u_data + stride/2;
rtc::scoped_refptr<I420BufferInterface> copy = webrtc::WrapI420Buffer(
width(), height(), y_data, stride, u_data, stride, v_data, stride,
rtc::Bind(&AlignedFree, yuv_data.release()));
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
// TODO(sakal): This call to a deperecated method will be removed when
// AndroidTextureBuffer is removed.
jmethodID transform_mid = GetMethodID(
jni,
GetObjectClass(jni, surface_texture_helper_),
"textureToYUV",
"(Ljava/nio/ByteBuffer;IIII[F)V");
jobject byte_buffer = jni->NewDirectByteBuffer(y_data, size);
jfloatArray sampling_matrix = native_handle_.sampling_matrix.ToJava(jni);
jni->CallVoidMethod(surface_texture_helper_,
transform_mid,
byte_buffer, width(), height(), stride,
native_handle_.oes_texture_id, sampling_matrix);
CHECK_EXCEPTION(jni) << "textureToYUV throwed an exception";
return copy;
}
rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBuffer::WrapReference(
JNIEnv* jni,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer) {
return new rtc::RefCountedObject<AndroidVideoBuffer>(
jni, j_release_id, width, height, j_video_frame_buffer);
}
AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni,
jmethodID j_retain_id,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer)
: AndroidVideoBuffer(jni,
j_release_id,
width,
height,
j_video_frame_buffer) {
jni->CallVoidMethod(j_video_frame_buffer, j_retain_id);
}
AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer)
: j_release_id_(j_release_id),
width_(width),
height_(height),
j_video_frame_buffer_(jni, j_video_frame_buffer) {}
AndroidVideoBuffer::~AndroidVideoBuffer() {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
jni->CallVoidMethod(*j_video_frame_buffer_, j_release_id_);
}
jobject AndroidVideoBuffer::video_frame_buffer() const {
return *j_video_frame_buffer_;
}
VideoFrameBuffer::Type AndroidVideoBuffer::type() const {
return Type::kNative;
}
int AndroidVideoBuffer::width() const {
return width_;
}
int AndroidVideoBuffer::height() const {
return height_;
}
rtc::scoped_refptr<I420BufferInterface> AndroidVideoBuffer::ToI420() {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
jclass j_video_frame_buffer_class =
FindClass(jni, "org/webrtc/VideoFrame$Buffer");
jmethodID j_to_i420_id =
jni->GetMethodID(j_video_frame_buffer_class, "toI420",
"()Lorg/webrtc/VideoFrame$I420Buffer;");
jobject j_i420_buffer =
jni->CallObjectMethod(*j_video_frame_buffer_, j_to_i420_id);
// We don't need to retain the buffer because toI420 returns a new object that
// we are assumed to take the ownership of.
return AndroidVideoI420Buffer::WrapReference(jni, j_release_id_, width_,
height_, j_i420_buffer);
}
jobject AndroidVideoBuffer::ToJavaI420Frame(JNIEnv* jni, int rotation) {
jclass j_byte_buffer_class = jni->FindClass("java/nio/ByteBuffer");
jclass j_i420_frame_class =
FindClass(jni, "org/webrtc/VideoRenderer$I420Frame");
jmethodID j_i420_frame_ctor_id = GetMethodID(
jni, j_i420_frame_class, "<init>", "(ILorg/webrtc/VideoFrame$Buffer;J)V");
// Java code just uses the native frame to hold a reference to the buffer so
// this is okay.
VideoFrame* native_frame =
new VideoFrame(this, 0 /* timestamp */, 0 /* render_time_ms */,
VideoRotation::kVideoRotation_0 /* rotation */);
return jni->NewObject(j_i420_frame_class, j_i420_frame_ctor_id, rotation,
*j_video_frame_buffer_, jlongFromPointer(native_frame));
}
AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni)
: j_video_frame_class_(jni, FindClass(jni, "org/webrtc/VideoFrame")),
j_get_buffer_id_(GetMethodID(jni,
*j_video_frame_class_,
"getBuffer",
"()Lorg/webrtc/VideoFrame$Buffer;")),
j_get_rotation_id_(
GetMethodID(jni, *j_video_frame_class_, "getRotation", "()I")),
j_get_timestamp_ns_id_(
GetMethodID(jni, *j_video_frame_class_, "getTimestampNs", "()J")),
j_video_frame_buffer_class_(
jni,
FindClass(jni, "org/webrtc/VideoFrame$Buffer")),
j_retain_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "retain", "()V")),
j_release_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "release", "()V")),
j_get_width_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "getWidth", "()I")),
j_get_height_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "getHeight", "()I")) {}
VideoFrame AndroidVideoBufferFactory::CreateFrame(
JNIEnv* jni,
jobject j_video_frame,
uint32_t timestamp_rtp) const {
jobject j_video_frame_buffer =
jni->CallObjectMethod(j_video_frame, j_get_buffer_id_);
int rotation = jni->CallIntMethod(j_video_frame, j_get_rotation_id_);
uint32_t timestamp_ns =
jni->CallLongMethod(j_video_frame, j_get_timestamp_ns_id_);
rtc::scoped_refptr<AndroidVideoBuffer> buffer =
CreateBuffer(jni, j_video_frame_buffer);
return VideoFrame(buffer, timestamp_rtp,
timestamp_ns / rtc::kNumNanosecsPerMillisec,
static_cast<VideoRotation>(rotation));
}
rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBufferFactory::WrapBuffer(
JNIEnv* jni,
jobject j_video_frame_buffer) const {
int width = jni->CallIntMethod(j_video_frame_buffer, j_get_width_id_);
int height = jni->CallIntMethod(j_video_frame_buffer, j_get_height_id_);
return AndroidVideoBuffer::WrapReference(jni, j_release_id_, width, height,
j_video_frame_buffer);
}
rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBufferFactory::CreateBuffer(
JNIEnv* jni,
jobject j_video_frame_buffer) const {
int width = jni->CallIntMethod(j_video_frame_buffer, j_get_width_id_);
int height = jni->CallIntMethod(j_video_frame_buffer, j_get_height_id_);
return new rtc::RefCountedObject<AndroidVideoBuffer>(
jni, j_retain_id_, j_release_id_, width, height, j_video_frame_buffer);
}
JavaVideoFrameFactory::JavaVideoFrameFactory(JNIEnv* jni)
: j_video_frame_class_(jni, FindClass(jni, "org/webrtc/VideoFrame")) {
j_video_frame_constructor_id_ =
GetMethodID(jni, *j_video_frame_class_, "<init>",
"(Lorg/webrtc/VideoFrame$Buffer;IJ)V");
}
static bool IsJavaVideoBuffer(rtc::scoped_refptr<VideoFrameBuffer> buffer) {
if (buffer->type() != VideoFrameBuffer::Type::kNative) {
return false;
}
AndroidVideoFrameBuffer* android_buffer =
static_cast<AndroidVideoFrameBuffer*>(buffer.get());
return android_buffer->android_type() ==
AndroidVideoFrameBuffer::AndroidType::kJavaBuffer;
}
jobject JavaVideoFrameFactory::ToJavaFrame(JNIEnv* jni,
const VideoFrame& frame) const {
rtc::scoped_refptr<VideoFrameBuffer> buffer = frame.video_frame_buffer();
jobject j_buffer;
if (IsJavaVideoBuffer(buffer)) {
RTC_DCHECK(buffer->type() == VideoFrameBuffer::Type::kNative);
AndroidVideoFrameBuffer* android_buffer =
static_cast<AndroidVideoFrameBuffer*>(buffer.get());
RTC_DCHECK(android_buffer->android_type() ==
AndroidVideoFrameBuffer::AndroidType::kJavaBuffer);
AndroidVideoBuffer* android_video_buffer =
static_cast<AndroidVideoBuffer*>(android_buffer);
j_buffer = android_video_buffer->video_frame_buffer();
} else {
j_buffer = WrapI420Buffer(jni, buffer->ToI420());
}
return jni->NewObject(
*j_video_frame_class_, j_video_frame_constructor_id_, j_buffer,
static_cast<jint>(frame.rotation()),
static_cast<jlong>(frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec));
}
} // namespace jni
} // namespace webrtc