blob: 1bc75b0c3ba34b680cbbb3f8a13a7c7ab08c5699 [file] [log] [blame]
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "sdk/android/src/jni/videodecoderwrapper.h"
#include "api/video/video_frame.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/vp8_header_parser.h"
#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
#include "rtc_base/logging.h"
#include "rtc_base/safe_conversions.h"
#include "rtc_base/timeutils.h"
#include "sdk/android/src/jni/classreferenceholder.h"
namespace webrtc {
namespace jni {
namespace {
template <typename Dst, typename Src>
inline rtc::Optional<Dst> cast_optional(const rtc::Optional<Src>& value) {
return value ? rtc::Optional<Dst>(rtc::dchecked_cast<Dst, Src>(*value))
: rtc::nullopt;
}
} // namespace
VideoDecoderWrapper::VideoDecoderWrapper(JNIEnv* jni, jobject decoder)
: decoder_(jni, decoder),
encoded_image_class_(jni, FindClass(jni, "org/webrtc/EncodedImage")),
frame_type_class_(jni,
FindClass(jni, "org/webrtc/EncodedImage$FrameType")),
settings_class_(jni, FindClass(jni, "org/webrtc/VideoDecoder$Settings")),
video_frame_class_(jni, FindClass(jni, "org/webrtc/VideoFrame")),
video_codec_status_class_(jni,
FindClass(jni, "org/webrtc/VideoCodecStatus")),
integer_class_(jni, jni->FindClass("java/lang/Integer")) {
encoded_image_constructor_ =
jni->GetMethodID(*encoded_image_class_, "<init>",
"(Ljava/nio/ByteBuffer;IIJLorg/webrtc/"
"EncodedImage$FrameType;IZLjava/lang/Integer;)V");
settings_constructor_ =
jni->GetMethodID(*settings_class_, "<init>", "(III)V");
empty_frame_field_ = jni->GetStaticFieldID(
*frame_type_class_, "EmptyFrame", "Lorg/webrtc/EncodedImage$FrameType;");
video_frame_key_field_ =
jni->GetStaticFieldID(*frame_type_class_, "VideoFrameKey",
"Lorg/webrtc/EncodedImage$FrameType;");
video_frame_delta_field_ =
jni->GetStaticFieldID(*frame_type_class_, "VideoFrameDelta",
"Lorg/webrtc/EncodedImage$FrameType;");
video_frame_get_timestamp_ns_method_ =
jni->GetMethodID(*video_frame_class_, "getTimestampNs", "()J");
jclass decoder_class = jni->GetObjectClass(decoder);
init_decode_method_ =
jni->GetMethodID(decoder_class, "initDecode",
"(Lorg/webrtc/VideoDecoder$Settings;Lorg/webrtc/"
"VideoDecoder$Callback;)Lorg/webrtc/VideoCodecStatus;");
release_method_ = jni->GetMethodID(decoder_class, "release",
"()Lorg/webrtc/VideoCodecStatus;");
decode_method_ = jni->GetMethodID(decoder_class, "decode",
"(Lorg/webrtc/EncodedImage;Lorg/webrtc/"
"VideoDecoder$DecodeInfo;)Lorg/webrtc/"
"VideoCodecStatus;");
get_prefers_late_decoding_method_ =
jni->GetMethodID(decoder_class, "getPrefersLateDecoding", "()Z");
get_implementation_name_method_ = jni->GetMethodID(
decoder_class, "getImplementationName", "()Ljava/lang/String;");
get_number_method_ =
jni->GetMethodID(*video_codec_status_class_, "getNumber", "()I");
initialized_ = false;
// QP parsing starts enabled and we disable it if the decoder provides frames.
qp_parsing_enabled_ = true;
implementation_name_ = GetImplementationName(jni);
}
int32_t VideoDecoderWrapper::InitDecode(const VideoCodec* codec_settings,
int32_t number_of_cores) {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
codec_settings_ = *codec_settings;
number_of_cores_ = number_of_cores;
return InitDecodeInternal(jni);
}
int32_t VideoDecoderWrapper::InitDecodeInternal(JNIEnv* jni) {
jobject settings =
jni->NewObject(*settings_class_, settings_constructor_, number_of_cores_,
codec_settings_.width, codec_settings_.height);
jclass callback_class =
FindClass(jni, "org/webrtc/VideoDecoderWrapperCallback");
jmethodID callback_constructor =
jni->GetMethodID(callback_class, "<init>", "(J)V");
jobject callback = jni->NewObject(callback_class, callback_constructor,
jlongFromPointer(this));
jobject ret =
jni->CallObjectMethod(*decoder_, init_decode_method_, settings, callback);
if (jni->CallIntMethod(ret, get_number_method_) == WEBRTC_VIDEO_CODEC_OK) {
initialized_ = true;
}
// The decoder was reinitialized so re-enable the QP parsing in case it stops
// providing QP values.
qp_parsing_enabled_ = true;
return HandleReturnCode(jni, ret);
}
int32_t VideoDecoderWrapper::Decode(
const EncodedImage& input_image,
bool missing_frames,
const RTPFragmentationHeader* fragmentation,
const CodecSpecificInfo* codec_specific_info,
int64_t render_time_ms) {
if (!initialized_) {
// Most likely initializing the codec failed.
return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
}
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
FrameExtraInfo frame_extra_info;
frame_extra_info.capture_time_ns =
input_image.capture_time_ms_ * rtc::kNumNanosecsPerMillisec;
frame_extra_info.timestamp_rtp = input_image._timeStamp;
frame_extra_info.qp =
qp_parsing_enabled_ ? ParseQP(input_image) : rtc::nullopt;
frame_extra_infos_.push_back(frame_extra_info);
jobject jinput_image =
ConvertEncodedImageToJavaEncodedImage(jni, input_image);
jobject ret =
jni->CallObjectMethod(*decoder_, decode_method_, jinput_image, nullptr);
return HandleReturnCode(jni, ret);
}
int32_t VideoDecoderWrapper::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t VideoDecoderWrapper::Release() {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
jobject ret = jni->CallObjectMethod(*decoder_, release_method_);
frame_extra_infos_.clear();
initialized_ = false;
return HandleReturnCode(jni, ret);
}
bool VideoDecoderWrapper::PrefersLateDecoding() const {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
return jni->CallBooleanMethod(*decoder_, get_prefers_late_decoding_method_);
}
const char* VideoDecoderWrapper::ImplementationName() const {
return implementation_name_.c_str();
}
void VideoDecoderWrapper::OnDecodedFrame(JNIEnv* jni,
jobject jframe,
jobject jdecode_time_ms,
jobject jqp) {
const jlong capture_time_ns =
jni->CallLongMethod(jframe, video_frame_get_timestamp_ns_method_);
FrameExtraInfo frame_extra_info;
do {
if (frame_extra_infos_.empty()) {
RTC_LOG(LS_WARNING) << "Java decoder produced an unexpected frame.";
return;
}
frame_extra_info = frame_extra_infos_.front();
frame_extra_infos_.pop_front();
// If the decoder might drop frames so iterate through the queue until we
// find a matching timestamp.
} while (frame_extra_info.capture_time_ns != capture_time_ns);
VideoFrame frame =
JavaToNativeFrame(jni, jframe, frame_extra_info.timestamp_rtp);
rtc::Optional<int32_t> decoding_time_ms =
JavaIntegerToOptionalInt(jni, jdecode_time_ms);
rtc::Optional<uint8_t> decoder_qp =
cast_optional<uint8_t, int32_t>(JavaIntegerToOptionalInt(jni, jqp));
// If the decoder provides QP values itself, no need to parse the bitstream.
// Enable QP parsing if decoder does not provide QP values itself.
qp_parsing_enabled_ = !decoder_qp.has_value();
callback_->Decoded(frame, decoding_time_ms,
decoder_qp ? decoder_qp : frame_extra_info.qp);
}
jobject VideoDecoderWrapper::ConvertEncodedImageToJavaEncodedImage(
JNIEnv* jni,
const EncodedImage& image) {
jobject buffer = jni->NewDirectByteBuffer(image._buffer, image._length);
jfieldID frame_type_field;
switch (image._frameType) {
case kEmptyFrame:
frame_type_field = empty_frame_field_;
break;
case kVideoFrameKey:
frame_type_field = video_frame_key_field_;
break;
case kVideoFrameDelta:
frame_type_field = video_frame_delta_field_;
break;
default:
RTC_NOTREACHED();
return nullptr;
}
jobject frame_type =
jni->GetStaticObjectField(*frame_type_class_, frame_type_field);
jobject qp = nullptr;
if (image.qp_ != -1) {
qp = JavaIntegerFromInt(jni, image.qp_);
}
return jni->NewObject(
*encoded_image_class_, encoded_image_constructor_, buffer,
static_cast<jint>(image._encodedWidth),
static_cast<jint>(image._encodedHeight),
static_cast<jlong>(image.capture_time_ms_ * rtc::kNumNanosecsPerMillisec),
frame_type, static_cast<jint>(image.rotation_), image._completeFrame, qp);
}
int32_t VideoDecoderWrapper::HandleReturnCode(JNIEnv* jni, jobject code) {
int32_t value = jni->CallIntMethod(code, get_number_method_);
if (value < 0) { // Any errors are represented by negative values.
// Reset the codec.
if (Release() == WEBRTC_VIDEO_CODEC_OK) {
InitDecodeInternal(jni);
}
RTC_LOG(LS_WARNING) << "Falling back to software decoder.";
return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
} else {
return value;
}
}
rtc::Optional<uint8_t> VideoDecoderWrapper::ParseQP(
const EncodedImage& input_image) {
if (input_image.qp_ != -1) {
return input_image.qp_;
}
rtc::Optional<uint8_t> qp;
switch (codec_settings_.codecType) {
case kVideoCodecVP8: {
int qp_int;
if (vp8::GetQp(input_image._buffer, input_image._length, &qp_int)) {
qp = qp_int;
}
break;
}
case kVideoCodecVP9: {
int qp_int;
if (vp9::GetQp(input_image._buffer, input_image._length, &qp_int)) {
qp = qp_int;
}
break;
}
case kVideoCodecH264: {
h264_bitstream_parser_.ParseBitstream(input_image._buffer,
input_image._length);
int qp_int;
if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) {
qp = qp_int;
}
break;
}
default:
break; // Default is to not provide QP.
}
return qp;
}
std::string VideoDecoderWrapper::GetImplementationName(JNIEnv* jni) const {
jstring jname = reinterpret_cast<jstring>(
jni->CallObjectMethod(*decoder_, get_implementation_name_method_));
return JavaToStdString(jni, jname);
}
JNI_FUNCTION_DECLARATION(void,
VideoDecoderWrapperCallback_nativeOnDecodedFrame,
JNIEnv* jni,
jclass,
jlong jnative_decoder,
jobject jframe,
jobject jdecode_time_ms,
jobject jqp) {
VideoDecoderWrapper* native_decoder =
reinterpret_cast<VideoDecoderWrapper*>(jnative_decoder);
native_decoder->OnDecodedFrame(jni, jframe, jdecode_time_ms, jqp);
}
} // namespace jni
} // namespace webrtc