Support native scaling of VideoFrameBuffers in LibvpxVp9Encoder.
This CL is part of Optimized Scaling efforts. In Chromium, the native
frame buffer is getting an optimized CropAndScale() implementation. To
support HW accelerated scaling, returning pre-scaled images and skipping
unnecessary intermediate downscales, WebRTC needs to 1) use CropAndScale
instead of libyuv::XXXXScale and 2) only map buffers it actually intends
to encode.
- To achieve this, WebRTC encoders are updated to map kNative video
buffers so that in a follow-up CL VideoStreamEncoder can stop mapping
intermediate buffer sizes.
In this CL LibvpxVp9Encoder is updated to map kNative buffers of pixel
formats it supports and convert ToI420() if the kNative buffer is
something else. A fake native buffer that keeps track of which
resolutions were mapped, MappableNativeBuffer, is added.
Because VP9 is currently an SVC encoder and not a simulcast encoder, it
does not need to invoke CropAndScale.
This CL also fixes MultiplexEncoderAdapter, but because it simply
forwards frames it only cares about the pixel format when
|supports_augmented_data_| is true so this is the only time we map it.
Because this encoder is not used with kNative in practise, we don't care
to make this path optimal.
Bug: webrtc:12469, chromium:1157072
Change-Id: I74edf85b18eccd0d250776bbade7a6444478efce
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/212580
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Cr-Commit-Position: refs/heads/master@{#33526}
diff --git a/test/mappable_native_buffer.cc b/test/mappable_native_buffer.cc
new file mode 100644
index 0000000..cff58ff
--- /dev/null
+++ b/test/mappable_native_buffer.cc
@@ -0,0 +1,186 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/mappable_native_buffer.h"
+
+#include "absl/algorithm/container.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/nv12_buffer.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_rotation.h"
+#include "common_video/include/video_frame_buffer.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace test {
+
+namespace {
+
+class NV12BufferWithDidConvertToI420 : public NV12Buffer {
+ public:
+ NV12BufferWithDidConvertToI420(int width, int height)
+ : NV12Buffer(width, height), did_convert_to_i420_(false) {}
+
+ bool did_convert_to_i420() const { return did_convert_to_i420_; }
+
+ rtc::scoped_refptr<I420BufferInterface> ToI420() override {
+ did_convert_to_i420_ = true;
+ return NV12Buffer::ToI420();
+ }
+
+ private:
+ bool did_convert_to_i420_;
+};
+
+} // namespace
+
+VideoFrame CreateMappableNativeFrame(int64_t ntp_time_ms,
+ VideoFrameBuffer::Type mappable_type,
+ int width,
+ int height) {
+ VideoFrame frame = VideoFrame::Builder()
+ .set_video_frame_buffer(
+ new rtc::RefCountedObject<MappableNativeBuffer>(
+ mappable_type, width, height))
+ .set_timestamp_rtp(99)
+ .set_timestamp_ms(99)
+ .set_rotation(kVideoRotation_0)
+ .build();
+ frame.set_ntp_time_ms(ntp_time_ms);
+ return frame;
+}
+
+rtc::scoped_refptr<MappableNativeBuffer> GetMappableNativeBufferFromVideoFrame(
+ const VideoFrame& frame) {
+ return static_cast<MappableNativeBuffer*>(frame.video_frame_buffer().get());
+}
+
+MappableNativeBuffer::ScaledBuffer::ScaledBuffer(
+ rtc::scoped_refptr<MappableNativeBuffer> parent,
+ int width,
+ int height)
+ : parent_(std::move(parent)), width_(width), height_(height) {}
+
+MappableNativeBuffer::ScaledBuffer::~ScaledBuffer() {}
+
+rtc::scoped_refptr<VideoFrameBuffer>
+MappableNativeBuffer::ScaledBuffer::CropAndScale(int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height,
+ int scaled_width,
+ int scaled_height) {
+ return rtc::scoped_refptr<VideoFrameBuffer>(
+ new rtc::RefCountedObject<ScaledBuffer>(parent_, scaled_width,
+ scaled_height));
+}
+
+rtc::scoped_refptr<I420BufferInterface>
+MappableNativeBuffer::ScaledBuffer::ToI420() {
+ return parent_->GetOrCreateMappedBuffer(width_, height_)->ToI420();
+}
+
+rtc::scoped_refptr<VideoFrameBuffer>
+MappableNativeBuffer::ScaledBuffer::GetMappedFrameBuffer(
+ rtc::ArrayView<VideoFrameBuffer::Type> types) {
+ if (absl::c_find(types, parent_->mappable_type_) == types.end())
+ return nullptr;
+ return parent_->GetOrCreateMappedBuffer(width_, height_);
+}
+
+MappableNativeBuffer::MappableNativeBuffer(VideoFrameBuffer::Type mappable_type,
+ int width,
+ int height)
+ : mappable_type_(mappable_type), width_(width), height_(height) {
+ RTC_DCHECK(mappable_type_ == VideoFrameBuffer::Type::kI420 ||
+ mappable_type_ == VideoFrameBuffer::Type::kNV12);
+}
+
+MappableNativeBuffer::~MappableNativeBuffer() {}
+
+rtc::scoped_refptr<VideoFrameBuffer> MappableNativeBuffer::CropAndScale(
+ int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height,
+ int scaled_width,
+ int scaled_height) {
+ return FullSizeBuffer()->CropAndScale(
+ offset_x, offset_y, crop_width, crop_height, scaled_width, scaled_height);
+}
+
+rtc::scoped_refptr<I420BufferInterface> MappableNativeBuffer::ToI420() {
+ return FullSizeBuffer()->ToI420();
+}
+
+rtc::scoped_refptr<VideoFrameBuffer> MappableNativeBuffer::GetMappedFrameBuffer(
+ rtc::ArrayView<VideoFrameBuffer::Type> types) {
+ return FullSizeBuffer()->GetMappedFrameBuffer(types);
+}
+
+std::vector<rtc::scoped_refptr<VideoFrameBuffer>>
+MappableNativeBuffer::GetMappedFramedBuffers() const {
+ MutexLock lock(&lock_);
+ return mapped_buffers_;
+}
+
+bool MappableNativeBuffer::DidConvertToI420() const {
+ if (mappable_type_ != VideoFrameBuffer::Type::kNV12)
+ return false;
+ MutexLock lock(&lock_);
+ for (auto& mapped_buffer : mapped_buffers_) {
+ if (static_cast<NV12BufferWithDidConvertToI420*>(mapped_buffer.get())
+ ->did_convert_to_i420()) {
+ return true;
+ }
+ }
+ return false;
+}
+
+rtc::scoped_refptr<MappableNativeBuffer::ScaledBuffer>
+MappableNativeBuffer::FullSizeBuffer() {
+ return rtc::scoped_refptr<MappableNativeBuffer::ScaledBuffer>(
+ new rtc::RefCountedObject<ScaledBuffer>(this, width_, height_));
+}
+
+rtc::scoped_refptr<VideoFrameBuffer>
+MappableNativeBuffer::GetOrCreateMappedBuffer(int width, int height) {
+ MutexLock lock(&lock_);
+ for (auto& mapped_buffer : mapped_buffers_) {
+ if (mapped_buffer->width() == width && mapped_buffer->height() == height) {
+ return mapped_buffer;
+ }
+ }
+ rtc::scoped_refptr<VideoFrameBuffer> mapped_buffer;
+ switch (mappable_type_) {
+ case VideoFrameBuffer::Type::kI420: {
+ rtc::scoped_refptr<I420Buffer> i420_buffer =
+ I420Buffer::Create(width, height);
+ I420Buffer::SetBlack(i420_buffer);
+ mapped_buffer = i420_buffer;
+ break;
+ }
+ case VideoFrameBuffer::Type::kNV12: {
+ rtc::scoped_refptr<NV12Buffer> nv12_buffer;
+ nv12_buffer = new rtc::RefCountedObject<NV12BufferWithDidConvertToI420>(
+ width, height);
+ nv12_buffer->InitializeData();
+ mapped_buffer = nv12_buffer;
+ break;
+ }
+ default:
+ RTC_NOTREACHED();
+ }
+ mapped_buffers_.push_back(mapped_buffer);
+ return mapped_buffer;
+}
+
+} // namespace test
+} // namespace webrtc
