blob: de8538e644df7d798604e36ee42d9522b46a590a [file] [log] [blame]
Erik Språngcc681cc2018-03-14 16:52:551/*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Yves Gerey3e707812018-11-28 15:47:4911#include <stdio.h>
12#include <string.h>
Erik Språngcc681cc2018-03-14 16:52:5513#include <algorithm>
14#include <string>
15
Karl Wiberg918f50c2018-07-05 09:40:3316#include "absl/memory/memory.h"
Yves Gerey3e707812018-11-28 15:47:4917#include "absl/types/optional.h"
18#include "api/video/i420_buffer.h"
19#include "api/video/video_frame.h"
20#include "api/video/video_frame_buffer.h"
21#include "api/video/video_rotation.h"
22#include "common_types.h" // NOLINT(build/include)
23#include "modules/video_coding/codecs/vp8/include/vp8.h"
Erik Språngcc681cc2018-03-14 16:52:5524#include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h"
Yves Gerey3e707812018-11-28 15:47:4925#include "modules/video_coding/include/video_error_codes.h"
Erik Språngcc681cc2018-03-14 16:52:5526#include "rtc_base/checks.h"
27#include "rtc_base/numerics/exp_filter.h"
Yves Gerey3e707812018-11-28 15:47:4928#include "rtc_base/scoped_ref_ptr.h"
Steve Anton10542f22019-01-11 17:11:0029#include "rtc_base/time_utils.h"
Erik Språngcc681cc2018-03-14 16:52:5530#include "system_wrappers/include/field_trial.h"
31#include "system_wrappers/include/metrics.h"
32#include "third_party/libyuv/include/libyuv/convert.h"
Yves Gerey3e707812018-11-28 15:47:4933#include "vpx/vp8.h"
34#include "vpx/vp8dx.h"
35#include "vpx/vpx_decoder.h"
Erik Språngcc681cc2018-03-14 16:52:5536
37namespace webrtc {
38namespace {
39constexpr int kVp8ErrorPropagationTh = 30;
40// vpx_decoder.h documentation indicates decode deadline is time in us, with
41// "Set to zero for unlimited.", but actual implementation requires this to be
42// a mode with 0 meaning allow delay and 1 not allowing it.
43constexpr long kDecodeDeadlineRealtime = 1; // NOLINT
44
45const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm";
46
47void GetPostProcParamsFromFieldTrialGroup(
48 LibvpxVp8Decoder::DeblockParams* deblock_params) {
49 std::string group =
50 webrtc::field_trial::FindFullName(kVp8PostProcArmFieldTrial);
51 if (group.empty())
52 return;
53
54 LibvpxVp8Decoder::DeblockParams params;
55 if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &params.max_level,
56 &params.min_qp, &params.degrade_qp) != 3)
57 return;
58
59 if (params.max_level < 0 || params.max_level > 16)
60 return;
61
62 if (params.min_qp < 0 || params.degrade_qp <= params.min_qp)
63 return;
64
65 *deblock_params = params;
66}
67
68} // namespace
69
Erik Språng5fbc0e02018-10-04 15:52:3670std::unique_ptr<VideoDecoder> VP8Decoder::Create() {
Karl Wiberg918f50c2018-07-05 09:40:3371 return absl::make_unique<LibvpxVp8Decoder>();
Erik Språngcc681cc2018-03-14 16:52:5572}
73
74class LibvpxVp8Decoder::QpSmoother {
75 public:
76 QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {}
77
78 int GetAvg() const {
79 float value = smoother_.filtered();
80 return (value == rtc::ExpFilter::kValueUndefined) ? 0
81 : static_cast<int>(value);
82 }
83
84 void Add(float sample) {
85 int64_t now_ms = rtc::TimeMillis();
86 smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
87 last_sample_ms_ = now_ms;
88 }
89
90 void Reset() { smoother_.Reset(kAlpha); }
91
92 private:
93 const float kAlpha = 0.95f;
94 int64_t last_sample_ms_;
95 rtc::ExpFilter smoother_;
96};
97
98LibvpxVp8Decoder::LibvpxVp8Decoder()
99 : use_postproc_arm_(
100 webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)),
101 buffer_pool_(false, 300 /* max_number_of_buffers*/),
102 decode_complete_callback_(NULL),
103 inited_(false),
104 decoder_(NULL),
105 propagation_cnt_(-1),
106 last_frame_width_(0),
107 last_frame_height_(0),
108 key_frame_required_(true),
109 qp_smoother_(use_postproc_arm_ ? new QpSmoother() : nullptr) {
110 if (use_postproc_arm_)
111 GetPostProcParamsFromFieldTrialGroup(&deblock_);
112}
113
114LibvpxVp8Decoder::~LibvpxVp8Decoder() {
115 inited_ = true; // in order to do the actual release
116 Release();
117}
118
119int LibvpxVp8Decoder::InitDecode(const VideoCodec* inst, int number_of_cores) {
120 int ret_val = Release();
121 if (ret_val < 0) {
122 return ret_val;
123 }
124 if (decoder_ == NULL) {
125 decoder_ = new vpx_codec_ctx_t;
126 memset(decoder_, 0, sizeof(*decoder_));
127 }
128 vpx_codec_dec_cfg_t cfg;
129 // Setting number of threads to a constant value (1)
130 cfg.threads = 1;
131 cfg.h = cfg.w = 0; // set after decode
132
133#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
134 defined(WEBRTC_ANDROID)
135 vpx_codec_flags_t flags = use_postproc_arm_ ? VPX_CODEC_USE_POSTPROC : 0;
136#else
137 vpx_codec_flags_t flags = VPX_CODEC_USE_POSTPROC;
138#endif
139
140 if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) {
141 delete decoder_;
142 decoder_ = nullptr;
143 return WEBRTC_VIDEO_CODEC_MEMORY;
144 }
145
146 propagation_cnt_ = -1;
147 inited_ = true;
148
149 // Always start with a complete key frame.
150 key_frame_required_ = true;
151 return WEBRTC_VIDEO_CODEC_OK;
152}
153
154int LibvpxVp8Decoder::Decode(const EncodedImage& input_image,
155 bool missing_frames,
Erik Språngcc681cc2018-03-14 16:52:55156 const CodecSpecificInfo* codec_specific_info,
157 int64_t /*render_time_ms*/) {
158 if (!inited_) {
159 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
160 }
161 if (decode_complete_callback_ == NULL) {
162 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
163 }
Niels Möller77536a22019-01-15 07:50:01164 if (input_image.data() == NULL && input_image.size() > 0) {
Erik Språngcc681cc2018-03-14 16:52:55165 // Reset to avoid requesting key frames too often.
166 if (propagation_cnt_ > 0)
167 propagation_cnt_ = 0;
168 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
169 }
170
171// Post process configurations.
172#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
173 defined(WEBRTC_ANDROID)
174 if (use_postproc_arm_) {
175 vp8_postproc_cfg_t ppcfg;
176 ppcfg.post_proc_flag = VP8_MFQE;
177 // For low resolutions, use stronger deblocking filter.
178 int last_width_x_height = last_frame_width_ * last_frame_height_;
179 if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) {
180 // Enable the deblock and demacroblocker based on qp thresholds.
181 RTC_DCHECK(qp_smoother_);
182 int qp = qp_smoother_->GetAvg();
183 if (qp > deblock_.min_qp) {
184 int level = deblock_.max_level;
185 if (qp < deblock_.degrade_qp) {
186 // Use lower level.
187 level = deblock_.max_level * (qp - deblock_.min_qp) /
188 (deblock_.degrade_qp - deblock_.min_qp);
189 }
190 // Deblocking level only affects VP8_DEMACROBLOCK.
191 ppcfg.deblocking_level = std::max(level, 1);
192 ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK;
193 }
194 }
195 vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
196 }
197#else
198 vp8_postproc_cfg_t ppcfg;
199 // MFQE enabled to reduce key frame popping.
200 ppcfg.post_proc_flag = VP8_MFQE | VP8_DEBLOCK;
201 // For VGA resolutions and lower, enable the demacroblocker postproc.
202 if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
203 ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
204 }
205 // Strength of deblocking filter. Valid range:[0,16]
206 ppcfg.deblocking_level = 3;
207 vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
208#endif
209
210 // Always start with a complete key frame.
211 if (key_frame_required_) {
212 if (input_image._frameType != kVideoFrameKey)
213 return WEBRTC_VIDEO_CODEC_ERROR;
214 // We have a key frame - is it complete?
215 if (input_image._completeFrame) {
216 key_frame_required_ = false;
217 } else {
218 return WEBRTC_VIDEO_CODEC_ERROR;
219 }
220 }
221 // Restrict error propagation using key frame requests.
222 // Reset on a key frame refresh.
223 if (input_image._frameType == kVideoFrameKey && input_image._completeFrame) {
224 propagation_cnt_ = -1;
225 // Start count on first loss.
226 } else if ((!input_image._completeFrame || missing_frames) &&
227 propagation_cnt_ == -1) {
228 propagation_cnt_ = 0;
229 }
230 if (propagation_cnt_ >= 0) {
231 propagation_cnt_++;
232 }
233
234 vpx_codec_iter_t iter = NULL;
235 vpx_image_t* img;
236 int ret;
237
238 // Check for missing frames.
239 if (missing_frames) {
240 // Call decoder with zero data length to signal missing frames.
241 if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) {
242 // Reset to avoid requesting key frames too often.
243 if (propagation_cnt_ > 0)
244 propagation_cnt_ = 0;
245 return WEBRTC_VIDEO_CODEC_ERROR;
246 }
247 img = vpx_codec_get_frame(decoder_, &iter);
248 iter = NULL;
249 }
250
Niels Möller24871e42019-01-17 10:31:13251 const uint8_t* buffer = input_image.data();
Niels Möller77536a22019-01-15 07:50:01252 if (input_image.size() == 0) {
Erik Språngcc681cc2018-03-14 16:52:55253 buffer = NULL; // Triggers full frame concealment.
254 }
Niels Möller77536a22019-01-15 07:50:01255 if (vpx_codec_decode(decoder_, buffer, input_image.size(), 0,
Erik Språngcc681cc2018-03-14 16:52:55256 kDecodeDeadlineRealtime)) {
257 // Reset to avoid requesting key frames too often.
258 if (propagation_cnt_ > 0) {
259 propagation_cnt_ = 0;
260 }
261 return WEBRTC_VIDEO_CODEC_ERROR;
262 }
263
264 img = vpx_codec_get_frame(decoder_, &iter);
265 int qp;
266 vpx_codec_err_t vpx_ret =
267 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
268 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
Johannes Krona1bec232018-12-14 12:55:23269 ret = ReturnFrame(img, input_image.Timestamp(), input_image.ntp_time_ms_, qp,
270 input_image.ColorSpace());
Erik Språngcc681cc2018-03-14 16:52:55271 if (ret != 0) {
272 // Reset to avoid requesting key frames too often.
273 if (ret < 0 && propagation_cnt_ > 0)
274 propagation_cnt_ = 0;
275 return ret;
276 }
277 // Check Vs. threshold
278 if (propagation_cnt_ > kVp8ErrorPropagationTh) {
279 // Reset to avoid requesting key frames too often.
280 propagation_cnt_ = 0;
281 return WEBRTC_VIDEO_CODEC_ERROR;
282 }
283 return WEBRTC_VIDEO_CODEC_OK;
284}
285
286int LibvpxVp8Decoder::ReturnFrame(const vpx_image_t* img,
287 uint32_t timestamp,
288 int64_t ntp_time_ms,
Johannes Krona1bec232018-12-14 12:55:23289 int qp,
290 const ColorSpace* explicit_color_space) {
Erik Språngcc681cc2018-03-14 16:52:55291 if (img == NULL) {
292 // Decoder OK and NULL image => No show frame
293 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
294 }
295 if (qp_smoother_) {
296 if (last_frame_width_ != static_cast<int>(img->d_w) ||
297 last_frame_height_ != static_cast<int>(img->d_h)) {
298 qp_smoother_->Reset();
299 }
300 qp_smoother_->Add(qp);
301 }
302 last_frame_width_ = img->d_w;
303 last_frame_height_ = img->d_h;
304 // Allocate memory for decoded image.
305 rtc::scoped_refptr<I420Buffer> buffer =
306 buffer_pool_.CreateBuffer(img->d_w, img->d_h);
307 if (!buffer.get()) {
308 // Pool has too many pending frames.
309 RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames",
310 1);
311 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
312 }
313
314 libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y],
315 img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],
316 img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V],
317 buffer->MutableDataY(), buffer->StrideY(),
318 buffer->MutableDataU(), buffer->StrideU(),
319 buffer->MutableDataV(), buffer->StrideV(), img->d_w,
320 img->d_h);
321
Johannes Krona1bec232018-12-14 12:55:23322 VideoFrame decoded_image = VideoFrame::Builder()
323 .set_video_frame_buffer(buffer)
324 .set_timestamp_rtp(timestamp)
325 .set_ntp_time_ms(ntp_time_ms)
326 .set_color_space(explicit_color_space)
327 .build();
Danil Chapovalov0040b662018-06-18 08:48:16328 decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
Erik Språngcc681cc2018-03-14 16:52:55329
330 return WEBRTC_VIDEO_CODEC_OK;
331}
332
333int LibvpxVp8Decoder::RegisterDecodeCompleteCallback(
334 DecodedImageCallback* callback) {
335 decode_complete_callback_ = callback;
336 return WEBRTC_VIDEO_CODEC_OK;
337}
338
339int LibvpxVp8Decoder::Release() {
340 int ret_val = WEBRTC_VIDEO_CODEC_OK;
341
342 if (decoder_ != NULL) {
343 if (inited_) {
344 if (vpx_codec_destroy(decoder_)) {
345 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
346 }
347 }
348 delete decoder_;
349 decoder_ = NULL;
350 }
351 buffer_pool_.Release();
352 inited_ = false;
353 return ret_val;
354}
355
356const char* LibvpxVp8Decoder::ImplementationName() const {
357 return "libvpx";
358}
359} // namespace webrtc