| /* |
| * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include <stdio.h> |
| #include <string.h> |
| #include <algorithm> |
| #include <string> |
| |
| #include "absl/memory/memory.h" |
| #include "absl/types/optional.h" |
| #include "api/scoped_refptr.h" |
| #include "api/video/i420_buffer.h" |
| #include "api/video/video_frame.h" |
| #include "api/video/video_frame_buffer.h" |
| #include "api/video/video_rotation.h" |
| #include "modules/video_coding/codecs/vp8/include/vp8.h" |
| #include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h" |
| #include "modules/video_coding/include/video_error_codes.h" |
| #include "rtc_base/checks.h" |
| #include "rtc_base/numerics/exp_filter.h" |
| #include "rtc_base/time_utils.h" |
| #include "system_wrappers/include/field_trial.h" |
| #include "system_wrappers/include/metrics.h" |
| #include "third_party/libyuv/include/libyuv/convert.h" |
| #include "vpx/vp8.h" |
| #include "vpx/vp8dx.h" |
| #include "vpx/vpx_decoder.h" |
| |
| namespace webrtc { |
| namespace { |
| constexpr int kVp8ErrorPropagationTh = 30; |
| // vpx_decoder.h documentation indicates decode deadline is time in us, with |
| // "Set to zero for unlimited.", but actual implementation requires this to be |
| // a mode with 0 meaning allow delay and 1 not allowing it. |
| constexpr long kDecodeDeadlineRealtime = 1; // NOLINT |
| |
| const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm"; |
| |
| void GetPostProcParamsFromFieldTrialGroup( |
| LibvpxVp8Decoder::DeblockParams* deblock_params) { |
| std::string group = |
| webrtc::field_trial::FindFullName(kVp8PostProcArmFieldTrial); |
| if (group.empty()) |
| return; |
| |
| LibvpxVp8Decoder::DeblockParams params; |
| if (sscanf(group.c_str(), "Enabled-%d,%d,%d", ¶ms.max_level, |
| ¶ms.min_qp, ¶ms.degrade_qp) != 3) |
| return; |
| |
| if (params.max_level < 0 || params.max_level > 16) |
| return; |
| |
| if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) |
| return; |
| |
| *deblock_params = params; |
| } |
| |
| } // namespace |
| |
| std::unique_ptr<VideoDecoder> VP8Decoder::Create() { |
| return absl::make_unique<LibvpxVp8Decoder>(); |
| } |
| |
| class LibvpxVp8Decoder::QpSmoother { |
| public: |
| QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {} |
| |
| int GetAvg() const { |
| float value = smoother_.filtered(); |
| return (value == rtc::ExpFilter::kValueUndefined) ? 0 |
| : static_cast<int>(value); |
| } |
| |
| void Add(float sample) { |
| int64_t now_ms = rtc::TimeMillis(); |
| smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample); |
| last_sample_ms_ = now_ms; |
| } |
| |
| void Reset() { smoother_.Reset(kAlpha); } |
| |
| private: |
| const float kAlpha = 0.95f; |
| int64_t last_sample_ms_; |
| rtc::ExpFilter smoother_; |
| }; |
| |
| LibvpxVp8Decoder::LibvpxVp8Decoder() |
| : use_postproc_arm_( |
| webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)), |
| buffer_pool_(false, 300 /* max_number_of_buffers*/), |
| decode_complete_callback_(NULL), |
| inited_(false), |
| decoder_(NULL), |
| propagation_cnt_(-1), |
| last_frame_width_(0), |
| last_frame_height_(0), |
| key_frame_required_(true), |
| qp_smoother_(use_postproc_arm_ ? new QpSmoother() : nullptr) { |
| if (use_postproc_arm_) |
| GetPostProcParamsFromFieldTrialGroup(&deblock_); |
| } |
| |
| LibvpxVp8Decoder::~LibvpxVp8Decoder() { |
| inited_ = true; // in order to do the actual release |
| Release(); |
| } |
| |
| int LibvpxVp8Decoder::InitDecode(const VideoCodec* inst, int number_of_cores) { |
| int ret_val = Release(); |
| if (ret_val < 0) { |
| return ret_val; |
| } |
| if (decoder_ == NULL) { |
| decoder_ = new vpx_codec_ctx_t; |
| memset(decoder_, 0, sizeof(*decoder_)); |
| } |
| vpx_codec_dec_cfg_t cfg; |
| // Setting number of threads to a constant value (1) |
| cfg.threads = 1; |
| cfg.h = cfg.w = 0; // set after decode |
| |
| #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ |
| defined(WEBRTC_ANDROID) |
| vpx_codec_flags_t flags = use_postproc_arm_ ? VPX_CODEC_USE_POSTPROC : 0; |
| #else |
| vpx_codec_flags_t flags = VPX_CODEC_USE_POSTPROC; |
| #endif |
| |
| if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) { |
| delete decoder_; |
| decoder_ = nullptr; |
| return WEBRTC_VIDEO_CODEC_MEMORY; |
| } |
| |
| propagation_cnt_ = -1; |
| inited_ = true; |
| |
| // Always start with a complete key frame. |
| key_frame_required_ = true; |
| return WEBRTC_VIDEO_CODEC_OK; |
| } |
| |
| int LibvpxVp8Decoder::Decode(const EncodedImage& input_image, |
| bool missing_frames, |
| const CodecSpecificInfo* codec_specific_info, |
| int64_t /*render_time_ms*/) { |
| if (!inited_) { |
| return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| } |
| if (decode_complete_callback_ == NULL) { |
| return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| } |
| if (input_image.data() == NULL && input_image.size() > 0) { |
| // Reset to avoid requesting key frames too often. |
| if (propagation_cnt_ > 0) |
| propagation_cnt_ = 0; |
| return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| } |
| |
| // Post process configurations. |
| #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ |
| defined(WEBRTC_ANDROID) |
| if (use_postproc_arm_) { |
| vp8_postproc_cfg_t ppcfg; |
| ppcfg.post_proc_flag = VP8_MFQE; |
| // For low resolutions, use stronger deblocking filter. |
| int last_width_x_height = last_frame_width_ * last_frame_height_; |
| if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) { |
| // Enable the deblock and demacroblocker based on qp thresholds. |
| RTC_DCHECK(qp_smoother_); |
| int qp = qp_smoother_->GetAvg(); |
| if (qp > deblock_.min_qp) { |
| int level = deblock_.max_level; |
| if (qp < deblock_.degrade_qp) { |
| // Use lower level. |
| level = deblock_.max_level * (qp - deblock_.min_qp) / |
| (deblock_.degrade_qp - deblock_.min_qp); |
| } |
| // Deblocking level only affects VP8_DEMACROBLOCK. |
| ppcfg.deblocking_level = std::max(level, 1); |
| ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK; |
| } |
| } |
| vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); |
| } |
| #else |
| vp8_postproc_cfg_t ppcfg; |
| // MFQE enabled to reduce key frame popping. |
| ppcfg.post_proc_flag = VP8_MFQE | VP8_DEBLOCK; |
| // For VGA resolutions and lower, enable the demacroblocker postproc. |
| if (last_frame_width_ * last_frame_height_ <= 640 * 360) { |
| ppcfg.post_proc_flag |= VP8_DEMACROBLOCK; |
| } |
| // Strength of deblocking filter. Valid range:[0,16] |
| ppcfg.deblocking_level = 3; |
| vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); |
| #endif |
| |
| // Always start with a complete key frame. |
| if (key_frame_required_) { |
| if (input_image._frameType != VideoFrameType::kVideoFrameKey) |
| return WEBRTC_VIDEO_CODEC_ERROR; |
| // We have a key frame - is it complete? |
| if (input_image._completeFrame) { |
| key_frame_required_ = false; |
| } else { |
| return WEBRTC_VIDEO_CODEC_ERROR; |
| } |
| } |
| // Restrict error propagation using key frame requests. |
| // Reset on a key frame refresh. |
| if (input_image._frameType == VideoFrameType::kVideoFrameKey && |
| input_image._completeFrame) { |
| propagation_cnt_ = -1; |
| // Start count on first loss. |
| } else if ((!input_image._completeFrame || missing_frames) && |
| propagation_cnt_ == -1) { |
| propagation_cnt_ = 0; |
| } |
| if (propagation_cnt_ >= 0) { |
| propagation_cnt_++; |
| } |
| |
| vpx_codec_iter_t iter = NULL; |
| vpx_image_t* img; |
| int ret; |
| |
| // Check for missing frames. |
| if (missing_frames) { |
| // Call decoder with zero data length to signal missing frames. |
| if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) { |
| // Reset to avoid requesting key frames too often. |
| if (propagation_cnt_ > 0) |
| propagation_cnt_ = 0; |
| return WEBRTC_VIDEO_CODEC_ERROR; |
| } |
| img = vpx_codec_get_frame(decoder_, &iter); |
| iter = NULL; |
| } |
| |
| const uint8_t* buffer = input_image.data(); |
| if (input_image.size() == 0) { |
| buffer = NULL; // Triggers full frame concealment. |
| } |
| if (vpx_codec_decode(decoder_, buffer, input_image.size(), 0, |
| kDecodeDeadlineRealtime)) { |
| // Reset to avoid requesting key frames too often. |
| if (propagation_cnt_ > 0) { |
| propagation_cnt_ = 0; |
| } |
| return WEBRTC_VIDEO_CODEC_ERROR; |
| } |
| |
| img = vpx_codec_get_frame(decoder_, &iter); |
| int qp; |
| vpx_codec_err_t vpx_ret = |
| vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp); |
| RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK); |
| ret = ReturnFrame(img, input_image.Timestamp(), input_image.ntp_time_ms_, qp, |
| input_image.ColorSpace()); |
| if (ret != 0) { |
| // Reset to avoid requesting key frames too often. |
| if (ret < 0 && propagation_cnt_ > 0) |
| propagation_cnt_ = 0; |
| return ret; |
| } |
| // Check Vs. threshold |
| if (propagation_cnt_ > kVp8ErrorPropagationTh) { |
| // Reset to avoid requesting key frames too often. |
| propagation_cnt_ = 0; |
| return WEBRTC_VIDEO_CODEC_ERROR; |
| } |
| return WEBRTC_VIDEO_CODEC_OK; |
| } |
| |
| int LibvpxVp8Decoder::ReturnFrame( |
| const vpx_image_t* img, |
| uint32_t timestamp, |
| int64_t ntp_time_ms, |
| int qp, |
| const webrtc::ColorSpace* explicit_color_space) { |
| if (img == NULL) { |
| // Decoder OK and NULL image => No show frame |
| return WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
| } |
| if (qp_smoother_) { |
| if (last_frame_width_ != static_cast<int>(img->d_w) || |
| last_frame_height_ != static_cast<int>(img->d_h)) { |
| qp_smoother_->Reset(); |
| } |
| qp_smoother_->Add(qp); |
| } |
| last_frame_width_ = img->d_w; |
| last_frame_height_ = img->d_h; |
| // Allocate memory for decoded image. |
| rtc::scoped_refptr<I420Buffer> buffer = |
| buffer_pool_.CreateBuffer(img->d_w, img->d_h); |
| if (!buffer.get()) { |
| // Pool has too many pending frames. |
| RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames", |
| 1); |
| return WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
| } |
| |
| libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y], |
| img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U], |
| img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V], |
| buffer->MutableDataY(), buffer->StrideY(), |
| buffer->MutableDataU(), buffer->StrideU(), |
| buffer->MutableDataV(), buffer->StrideV(), img->d_w, |
| img->d_h); |
| |
| VideoFrame decoded_image = VideoFrame::Builder() |
| .set_video_frame_buffer(buffer) |
| .set_timestamp_rtp(timestamp) |
| .set_ntp_time_ms(ntp_time_ms) |
| .set_color_space(explicit_color_space) |
| .build(); |
| decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp); |
| |
| return WEBRTC_VIDEO_CODEC_OK; |
| } |
| |
| int LibvpxVp8Decoder::RegisterDecodeCompleteCallback( |
| DecodedImageCallback* callback) { |
| decode_complete_callback_ = callback; |
| return WEBRTC_VIDEO_CODEC_OK; |
| } |
| |
| int LibvpxVp8Decoder::Release() { |
| int ret_val = WEBRTC_VIDEO_CODEC_OK; |
| |
| if (decoder_ != NULL) { |
| if (inited_) { |
| if (vpx_codec_destroy(decoder_)) { |
| ret_val = WEBRTC_VIDEO_CODEC_MEMORY; |
| } |
| } |
| delete decoder_; |
| decoder_ = NULL; |
| } |
| buffer_pool_.Release(); |
| inited_ = false; |
| return ret_val; |
| } |
| |
| const char* LibvpxVp8Decoder::ImplementationName() const { |
| return "libvpx"; |
| } |
| } // namespace webrtc |