Added support for H264 YUV444 (I444) decoding.
Added Nutanix Inc. to the AUTHORS file.
Bug: chromium:1251096
Change-Id: Ib47c2b1f94797afb6c5090f3c46eae6f13110992
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/234540
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35200}
diff --git a/AUTHORS b/AUTHORS
index cac58ca..4bb2cbb 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -135,6 +135,7 @@
MIPS Technologies <*@mips.com>
Mozilla Foundation <*@mozilla.com>
Netgem S.A. <*@netgem.com>
+Nutanix Inc. <*@nutanix.com>
NVIDIA Corporation <*@nvidia.com>
Opera Software ASA <*@opera.com>
Optical Tone Ltd <*@opticaltone.com>
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index e6052fe..9fd2891 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -43,6 +43,8 @@
sources = [
"i420_buffer.cc",
"i420_buffer.h",
+ "i444_buffer.cc",
+ "i444_buffer.h",
"nv12_buffer.cc",
"nv12_buffer.h",
"video_codec_type.h",
diff --git a/api/video/DEPS b/api/video/DEPS
index cf6770d..5a3e496 100644
--- a/api/video/DEPS
+++ b/api/video/DEPS
@@ -18,6 +18,10 @@
"+rtc_base/memory/aligned_malloc.h",
],
+ "i444_buffer\.h": [
+ "+rtc_base/memory/aligned_malloc.h",
+ ],
+
"nv12_buffer\.h": [
"+rtc_base/memory/aligned_malloc.h",
],
diff --git a/api/video/i444_buffer.cc b/api/video/i444_buffer.cc
new file mode 100644
index 0000000..5f8b714
--- /dev/null
+++ b/api/video/i444_buffer.cc
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/video/i444_buffer.h"
+
+#include <string.h>
+
+#include <algorithm>
+#include <utility>
+
+#include "api/video/i420_buffer.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/ref_counted_object.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+#include "third_party/libyuv/include/libyuv/planar_functions.h"
+#include "third_party/libyuv/include/libyuv/scale.h"
+
+// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD.
+static const int kBufferAlignment = 64;
+
+namespace webrtc {
+
+namespace {
+
+int I444DataSize(int height, int stride_y, int stride_u, int stride_v) {
+ return stride_y * height + stride_u * height + stride_v * height;
+}
+
+} // namespace
+
+I444Buffer::I444Buffer(int width, int height)
+ : I444Buffer(width, height, width, (width), (width)) {}
+
+I444Buffer::I444Buffer(int width,
+ int height,
+ int stride_y,
+ int stride_u,
+ int stride_v)
+ : width_(width),
+ height_(height),
+ stride_y_(stride_y),
+ stride_u_(stride_u),
+ stride_v_(stride_v),
+ data_(static_cast<uint8_t*>(
+ AlignedMalloc(I444DataSize(height, stride_y, stride_u, stride_v),
+ kBufferAlignment))) {
+ RTC_DCHECK_GT(width, 0);
+ RTC_DCHECK_GT(height, 0);
+ RTC_DCHECK_GE(stride_y, width);
+ RTC_DCHECK_GE(stride_u, (width));
+ RTC_DCHECK_GE(stride_v, (width));
+}
+
+I444Buffer::~I444Buffer() {}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Create(int width, int height) {
+ return new rtc::RefCountedObject<I444Buffer>(width, height);
+}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Create(int width,
+ int height,
+ int stride_y,
+ int stride_u,
+ int stride_v) {
+ return new rtc::RefCountedObject<I444Buffer>(width, height, stride_y,
+ stride_u, stride_v);
+}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Copy(
+ const I444BufferInterface& source) {
+ return Copy(source.width(), source.height(), source.DataY(), source.StrideY(),
+ source.DataU(), source.StrideU(), source.DataV(),
+ source.StrideV());
+}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Copy(int width,
+ int height,
+ const uint8_t* data_y,
+ int stride_y,
+ const uint8_t* data_u,
+ int stride_u,
+ const uint8_t* data_v,
+ int stride_v) {
+ // Note: May use different strides than the input data.
+ rtc::scoped_refptr<I444Buffer> buffer = Create(width, height);
+ RTC_CHECK_EQ(0, libyuv::I444Copy(data_y, stride_y, data_u, stride_u, data_v,
+ stride_v, buffer->MutableDataY(),
+ buffer->StrideY(), buffer->MutableDataU(),
+ buffer->StrideU(), buffer->MutableDataV(),
+ buffer->StrideV(), width, height));
+ return buffer;
+}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Rotate(
+ const I444BufferInterface& src,
+ VideoRotation rotation) {
+ RTC_CHECK(src.DataY());
+ RTC_CHECK(src.DataU());
+ RTC_CHECK(src.DataV());
+
+ int rotated_width = src.width();
+ int rotated_height = src.height();
+ if (rotation == webrtc::kVideoRotation_90 ||
+ rotation == webrtc::kVideoRotation_270) {
+ std::swap(rotated_width, rotated_height);
+ }
+
+ rtc::scoped_refptr<webrtc::I444Buffer> buffer =
+ I444Buffer::Create(rotated_width, rotated_height);
+
+ RTC_CHECK_EQ(0,
+ libyuv::I444Rotate(
+ src.DataY(), src.StrideY(), src.DataU(), src.StrideU(),
+ src.DataV(), src.StrideV(), buffer->MutableDataY(),
+ buffer->StrideY(), buffer->MutableDataU(), buffer->StrideU(),
+ buffer->MutableDataV(), buffer->StrideV(), src.width(),
+ src.height(), static_cast<libyuv::RotationMode>(rotation)));
+
+ return buffer;
+}
+
+rtc::scoped_refptr<I420BufferInterface> I444Buffer::ToI420() {
+ rtc::scoped_refptr<I420Buffer> i420_buffer =
+ I420Buffer::Create(width(), height());
+ libyuv::I444ToI420(DataY(), StrideY(), DataU(), StrideU(), DataV(), StrideV(),
+ i420_buffer->MutableDataY(), i420_buffer->StrideY(),
+ i420_buffer->MutableDataU(), i420_buffer->StrideU(),
+ i420_buffer->MutableDataV(), i420_buffer->StrideV(),
+ width(), height());
+ return i420_buffer;
+}
+
+void I444Buffer::InitializeData() {
+ memset(data_.get(), 0,
+ I444DataSize(height_, stride_y_, stride_u_, stride_v_));
+}
+
+int I444Buffer::width() const {
+ return width_;
+}
+
+int I444Buffer::height() const {
+ return height_;
+}
+
+const uint8_t* I444Buffer::DataY() const {
+ return data_.get();
+}
+const uint8_t* I444Buffer::DataU() const {
+ return data_.get() + stride_y_ * height_;
+}
+const uint8_t* I444Buffer::DataV() const {
+ return data_.get() + stride_y_ * height_ + stride_u_ * ((height_));
+}
+
+int I444Buffer::StrideY() const {
+ return stride_y_;
+}
+int I444Buffer::StrideU() const {
+ return stride_u_;
+}
+int I444Buffer::StrideV() const {
+ return stride_v_;
+}
+
+uint8_t* I444Buffer::MutableDataY() {
+ return const_cast<uint8_t*>(DataY());
+}
+uint8_t* I444Buffer::MutableDataU() {
+ return const_cast<uint8_t*>(DataU());
+}
+uint8_t* I444Buffer::MutableDataV() {
+ return const_cast<uint8_t*>(DataV());
+}
+
+void I444Buffer::CropAndScaleFrom(const I444BufferInterface& src,
+ int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height) {
+ RTC_CHECK_LE(crop_width, src.width());
+ RTC_CHECK_LE(crop_height, src.height());
+ RTC_CHECK_LE(crop_width + offset_x, src.width());
+ RTC_CHECK_LE(crop_height + offset_y, src.height());
+ RTC_CHECK_GE(offset_x, 0);
+ RTC_CHECK_GE(offset_y, 0);
+
+ const uint8_t* y_plane = src.DataY() + src.StrideY() * offset_y + offset_x;
+ const uint8_t* u_plane = src.DataU() + src.StrideU() * offset_y + offset_x;
+ const uint8_t* v_plane = src.DataV() + src.StrideV() * offset_y + offset_x;
+ int res =
+ libyuv::I444Scale(y_plane, src.StrideY(), u_plane, src.StrideU(), v_plane,
+ src.StrideV(), crop_width, crop_height, MutableDataY(),
+ StrideY(), MutableDataU(), StrideU(), MutableDataV(),
+ StrideV(), width(), height(), libyuv::kFilterBox);
+
+ RTC_DCHECK_EQ(res, 0);
+}
+
+} // namespace webrtc
diff --git a/api/video/i444_buffer.h b/api/video/i444_buffer.h
new file mode 100644
index 0000000..d46c245
--- /dev/null
+++ b/api/video/i444_buffer.h
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_I444_BUFFER_H_
+#define API_VIDEO_I444_BUFFER_H_
+
+#include <stdint.h>
+
+#include <memory>
+
+#include "api/scoped_refptr.h"
+#include "api/video/video_frame_buffer.h"
+#include "api/video/video_rotation.h"
+#include "rtc_base/memory/aligned_malloc.h"
+#include "rtc_base/system/rtc_export.h"
+
+namespace webrtc {
+
+// Plain I444 buffer in standard memory.
+// I444 represents an image with in YUV format withouth any chroma subsampling.
+// https://en.wikipedia.org/wiki/Chroma_subsampling#4:4:4
+class RTC_EXPORT I444Buffer : public I444BufferInterface {
+ public:
+ static rtc::scoped_refptr<I444Buffer> Create(int width, int height);
+ static rtc::scoped_refptr<I444Buffer> Create(int width,
+ int height,
+ int stride_y,
+ int stride_u,
+ int stride_v);
+
+ // Create a new buffer and copy the pixel data.
+ static rtc::scoped_refptr<I444Buffer> Copy(const I444BufferInterface& buffer);
+
+ static rtc::scoped_refptr<I444Buffer> Copy(int width,
+ int height,
+ const uint8_t* data_y,
+ int stride_y,
+ const uint8_t* data_u,
+ int stride_u,
+ const uint8_t* data_v,
+ int stride_v);
+
+ // Returns a rotated copy of |src|.
+ static rtc::scoped_refptr<I444Buffer> Rotate(const I444BufferInterface& src,
+ VideoRotation rotation);
+
+ rtc::scoped_refptr<I420BufferInterface> ToI420() final;
+ const I420BufferInterface* GetI420() const final { return nullptr; }
+
+ // Sets all three planes to all zeros. Used to work around for
+ // quirks in memory checkers
+ // (https://bugs.chromium.org/p/libyuv/issues/detail?id=377) and
+ // ffmpeg (http://crbug.com/390941).
+ // TODO(nisse): Deprecated. Should be deleted if/when those issues
+ // are resolved in a better way. Or in the mean time, use SetBlack.
+ void InitializeData();
+
+ int width() const override;
+ int height() const override;
+ const uint8_t* DataY() const override;
+ const uint8_t* DataU() const override;
+ const uint8_t* DataV() const override;
+
+ int StrideY() const override;
+ int StrideU() const override;
+ int StrideV() const override;
+
+ uint8_t* MutableDataY();
+ uint8_t* MutableDataU();
+ uint8_t* MutableDataV();
+
+ // Scale the cropped area of |src| to the size of |this| buffer, and
+ // write the result into |this|.
+ void CropAndScaleFrom(const I444BufferInterface& src,
+ int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height);
+
+ protected:
+ I444Buffer(int width, int height);
+ I444Buffer(int width, int height, int stride_y, int stride_u, int stride_v);
+
+ ~I444Buffer() override;
+
+ private:
+ const int width_;
+ const int height_;
+ const int stride_y_;
+ const int stride_u_;
+ const int stride_v_;
+ const std::unique_ptr<uint8_t, AlignedFreeDeleter> data_;
+};
+
+} // namespace webrtc
+
+#endif // API_VIDEO_I444_BUFFER_H_
diff --git a/api/video/test/BUILD.gn b/api/video/test/BUILD.gn
index 1573e78..5b0d57b 100644
--- a/api/video/test/BUILD.gn
+++ b/api/video/test/BUILD.gn
@@ -12,6 +12,7 @@
testonly = true
sources = [
"color_space_unittest.cc",
+ "i444_buffer_unittest.cc",
"nv12_buffer_unittest.cc",
"video_adaptation_counters_unittest.cc",
"video_bitrate_allocation_unittest.cc",
diff --git a/api/video/test/i444_buffer_unittest.cc b/api/video/test/i444_buffer_unittest.cc
new file mode 100644
index 0000000..52ce49a
--- /dev/null
+++ b/api/video/test/i444_buffer_unittest.cc
@@ -0,0 +1,112 @@
+
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/i444_buffer.h"
+
+#include "api/video/i420_buffer.h"
+#include "test/frame_utils.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+namespace {
+int GetY(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
+ return buf->DataY()[row * buf->StrideY() + col];
+}
+
+int GetU(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
+ return buf->DataU()[row * buf->StrideU() + col];
+}
+
+int GetV(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
+ return buf->DataV()[row * buf->StrideV() + col];
+}
+
+void FillI444Buffer(rtc::scoped_refptr<I444Buffer> buf) {
+ const uint8_t Y = 1;
+ const uint8_t U = 2;
+ const uint8_t V = 3;
+ for (int row = 0; row < buf->height(); ++row) {
+ for (int col = 0; col < buf->width(); ++col) {
+ buf->MutableDataY()[row * buf->StrideY() + col] = Y;
+ buf->MutableDataU()[row * buf->StrideU() + col] = U;
+ buf->MutableDataV()[row * buf->StrideV() + col] = V;
+ }
+ }
+}
+
+} // namespace
+
+TEST(I444BufferTest, InitialData) {
+ constexpr int stride = 3;
+ constexpr int width = 3;
+ constexpr int height = 3;
+
+ rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
+ EXPECT_EQ(width, i444_buffer->width());
+ EXPECT_EQ(height, i444_buffer->height());
+ EXPECT_EQ(stride, i444_buffer->StrideY());
+ EXPECT_EQ(stride, i444_buffer->StrideU());
+ EXPECT_EQ(stride, i444_buffer->StrideV());
+ EXPECT_EQ(3, i444_buffer->ChromaWidth());
+ EXPECT_EQ(3, i444_buffer->ChromaHeight());
+}
+
+TEST(I444BufferTest, ReadPixels) {
+ constexpr int width = 3;
+ constexpr int height = 3;
+
+ rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
+ // Y = 1, U = 2, V = 3.
+ FillI444Buffer(i444_buffer);
+ for (int row = 0; row < height; row++) {
+ for (int col = 0; col < width; col++) {
+ EXPECT_EQ(1, GetY(i444_buffer, col, row));
+ EXPECT_EQ(2, GetU(i444_buffer, col, row));
+ EXPECT_EQ(3, GetV(i444_buffer, col, row));
+ }
+ }
+}
+
+TEST(I444BufferTest, ToI420) {
+ constexpr int width = 3;
+ constexpr int height = 3;
+ constexpr int size_y = width * height;
+ constexpr int size_u = (width + 1) / 2 * (height + 1) / 2;
+ constexpr int size_v = (width + 1) / 2 * (height + 1) / 2;
+ rtc::scoped_refptr<I420Buffer> reference(I420Buffer::Create(width, height));
+ memset(reference->MutableDataY(), 8, size_y);
+ memset(reference->MutableDataU(), 4, size_u);
+ memset(reference->MutableDataV(), 2, size_v);
+
+ rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
+ // Convert the reference buffer to I444.
+ memset(i444_buffer->MutableDataY(), 8, size_y);
+ memset(i444_buffer->MutableDataU(), 4, size_y);
+ memset(i444_buffer->MutableDataV(), 2, size_y);
+
+ // Confirm YUV values are as expected.
+ for (int row = 0; row < height; row++) {
+ for (int col = 0; col < width; col++) {
+ EXPECT_EQ(8, GetY(i444_buffer, col, row));
+ EXPECT_EQ(4, GetU(i444_buffer, col, row));
+ EXPECT_EQ(2, GetV(i444_buffer, col, row));
+ }
+ }
+
+ rtc::scoped_refptr<I420BufferInterface> i420_buffer(i444_buffer->ToI420());
+ EXPECT_EQ(height, i420_buffer->height());
+ EXPECT_EQ(width, i420_buffer->width());
+ EXPECT_TRUE(test::FrameBufsEqual(reference, i420_buffer));
+}
+
+} // namespace webrtc
diff --git a/api/video/video_frame_buffer.cc b/api/video/video_frame_buffer.cc
index 7085010..cf31e9d 100644
--- a/api/video/video_frame_buffer.cc
+++ b/api/video/video_frame_buffer.cc
@@ -11,6 +11,7 @@
#include "api/video/video_frame_buffer.h"
#include "api/video/i420_buffer.h"
+#include "api/video/i444_buffer.h"
#include "api/video/nv12_buffer.h"
#include "rtc_base/checks.h"
@@ -117,6 +118,19 @@
return height();
}
+rtc::scoped_refptr<VideoFrameBuffer> I444BufferInterface::CropAndScale(
+ int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height,
+ int scaled_width,
+ int scaled_height) {
+ rtc::scoped_refptr<I444Buffer> result =
+ I444Buffer::Create(scaled_width, scaled_height);
+ result->CropAndScaleFrom(*this, offset_x, offset_y, crop_width, crop_height);
+ return result;
+}
+
VideoFrameBuffer::Type I010BufferInterface::type() const {
return Type::kI010;
}
diff --git a/api/video/video_frame_buffer.h b/api/video/video_frame_buffer.h
index 7b0782f..6098a481 100644
--- a/api/video/video_frame_buffer.h
+++ b/api/video/video_frame_buffer.h
@@ -184,6 +184,13 @@
int ChromaWidth() const final;
int ChromaHeight() const final;
+ rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height,
+ int scaled_width,
+ int scaled_height) override;
+
protected:
~I444BufferInterface() override {}
};
diff --git a/api/video_codecs/h264_profile_level_id.cc b/api/video_codecs/h264_profile_level_id.cc
index fa47758..02b43ba 100644
--- a/api/video_codecs/h264_profile_level_id.cc
+++ b/api/video_codecs/h264_profile_level_id.cc
@@ -68,7 +68,8 @@
{0x58, BitPattern("10xx0000"), H264Profile::kProfileBaseline},
{0x4D, BitPattern("0x0x0000"), H264Profile::kProfileMain},
{0x64, BitPattern("00000000"), H264Profile::kProfileHigh},
- {0x64, BitPattern("00001100"), H264Profile::kProfileConstrainedHigh}};
+ {0x64, BitPattern("00001100"), H264Profile::kProfileConstrainedHigh},
+ {0xF4, BitPattern("00000000"), H264Profile::kProfilePredictiveHigh444}};
struct LevelConstraint {
const int max_macroblocks_per_second;
@@ -228,6 +229,9 @@
case H264Profile::kProfileHigh:
profile_idc_iop_string = "6400";
break;
+ case H264Profile::kProfilePredictiveHigh444:
+ profile_idc_iop_string = "f400";
+ break;
// Unrecognized profile.
default:
return absl::nullopt;
diff --git a/api/video_codecs/h264_profile_level_id.h b/api/video_codecs/h264_profile_level_id.h
index 51d025c..4b46ad3 100644
--- a/api/video_codecs/h264_profile_level_id.h
+++ b/api/video_codecs/h264_profile_level_id.h
@@ -25,6 +25,7 @@
kProfileMain,
kProfileConstrainedHigh,
kProfileHigh,
+ kProfilePredictiveHigh444,
};
// All values are equal to ten times the level number, except level 1b which is
diff --git a/common_video/include/video_frame_buffer_pool.h b/common_video/include/video_frame_buffer_pool.h
index 539a6cc..f26a9f7 100644
--- a/common_video/include/video_frame_buffer_pool.h
+++ b/common_video/include/video_frame_buffer_pool.h
@@ -17,6 +17,7 @@
#include "api/scoped_refptr.h"
#include "api/video/i420_buffer.h"
+#include "api/video/i444_buffer.h"
#include "api/video/nv12_buffer.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/ref_counted_object.h"
@@ -43,6 +44,7 @@
// and there are less than `max_number_of_buffers` pending, a buffer is
// created. Returns null otherwise.
rtc::scoped_refptr<I420Buffer> CreateI420Buffer(int width, int height);
+ rtc::scoped_refptr<I444Buffer> CreateI444Buffer(int width, int height);
rtc::scoped_refptr<NV12Buffer> CreateNV12Buffer(int width, int height);
// Changes the max amount of buffers in the pool to the new value.
diff --git a/common_video/video_frame_buffer_pool.cc b/common_video/video_frame_buffer_pool.cc
index a450bd1..cddb7c5 100644
--- a/common_video/video_frame_buffer_pool.cc
+++ b/common_video/video_frame_buffer_pool.cc
@@ -20,12 +20,17 @@
bool HasOneRef(const rtc::scoped_refptr<VideoFrameBuffer>& buffer) {
// Cast to rtc::RefCountedObject is safe because this function is only called
// on locally created VideoFrameBuffers, which are either
- // `rtc::RefCountedObject<I420Buffer>` or `rtc::RefCountedObject<NV12Buffer>`.
+ // `rtc::RefCountedObject<I420Buffer>`, `rtc::RefCountedObject<I444Buffer>` or
+ // `rtc::RefCountedObject<NV12Buffer>`.
switch (buffer->type()) {
case VideoFrameBuffer::Type::kI420: {
return static_cast<rtc::RefCountedObject<I420Buffer>*>(buffer.get())
->HasOneRef();
}
+ case VideoFrameBuffer::Type::kI444: {
+ return static_cast<rtc::RefCountedObject<I444Buffer>*>(buffer.get())
+ ->HasOneRef();
+ }
case VideoFrameBuffer::Type::kNV12: {
return static_cast<rtc::RefCountedObject<NV12Buffer>*>(buffer.get())
->HasOneRef();
@@ -116,6 +121,37 @@
return buffer;
}
+rtc::scoped_refptr<I444Buffer> VideoFrameBufferPool::CreateI444Buffer(
+ int width,
+ int height) {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
+
+ rtc::scoped_refptr<VideoFrameBuffer> existing_buffer =
+ GetExistingBuffer(width, height, VideoFrameBuffer::Type::kI444);
+ if (existing_buffer) {
+ // Cast is safe because the only way kI444 buffer is created is
+ // in the same function below, where |RefCountedObject<I444Buffer>|
+ // is created.
+ rtc::RefCountedObject<I444Buffer>* raw_buffer =
+ static_cast<rtc::RefCountedObject<I444Buffer>*>(existing_buffer.get());
+ // Creates a new scoped_refptr, which is also pointing to the same
+ // RefCountedObject as buffer, increasing ref count.
+ return rtc::scoped_refptr<I444Buffer>(raw_buffer);
+ }
+
+ if (buffers_.size() >= max_number_of_buffers_)
+ return nullptr;
+ // Allocate new buffer.
+ rtc::scoped_refptr<I444Buffer> buffer =
+ new rtc::RefCountedObject<I444Buffer>(width, height);
+
+ if (zero_initialize_)
+ buffer->InitializeData();
+
+ buffers_.push_back(buffer);
+ return buffer;
+}
+
rtc::scoped_refptr<NV12Buffer> VideoFrameBufferPool::CreateNV12Buffer(
int width,
int height) {
diff --git a/modules/video_coding/codecs/h264/h264.cc b/modules/video_coding/codecs/h264/h264.cc
index 275e634..eef4dde 100644
--- a/modules/video_coding/codecs/h264/h264.cc
+++ b/modules/video_coding/codecs/h264/h264.cc
@@ -87,6 +87,10 @@
CreateH264Format(H264Profile::kProfileConstrainedBaseline,
H264Level::kLevel3_1, "1"),
CreateH264Format(H264Profile::kProfileConstrainedBaseline,
+ H264Level::kLevel3_1, "0"),
+ CreateH264Format(H264Profile::kProfilePredictiveHigh444,
+ H264Level::kLevel3_1, "1"),
+ CreateH264Format(H264Profile::kProfilePredictiveHigh444,
H264Level::kLevel3_1, "0")};
}
diff --git a/modules/video_coding/codecs/h264/h264_decoder_impl.cc b/modules/video_coding/codecs/h264/h264_decoder_impl.cc
index b42aac5..31279b7 100644
--- a/modules/video_coding/codecs/h264/h264_decoder_impl.cc
+++ b/modules/video_coding/codecs/h264/h264_decoder_impl.cc
@@ -41,8 +41,10 @@
namespace {
-const AVPixelFormat kPixelFormatDefault = AV_PIX_FMT_YUV420P;
-const AVPixelFormat kPixelFormatFullRange = AV_PIX_FMT_YUVJ420P;
+constexpr std::array<AVPixelFormat, 2> kPixelFormatsDefault = {
+ AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV444P};
+constexpr std::array<AVPixelFormat, 2> kPixelFormatsFullRange = {
+ AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ444P};
const size_t kYPlaneIndex = 0;
const size_t kUPlaneIndex = 1;
const size_t kVPlaneIndex = 2;
@@ -76,9 +78,17 @@
// Necessary capability to be allowed to provide our own buffers.
RTC_DCHECK(context->codec->capabilities | AV_CODEC_CAP_DR1);
+ // Limited or full range YUV420 or YUV444 is expected.
+ auto pixelFormatDefault = std::find_if(
+ kPixelFormatsDefault.begin(), kPixelFormatsDefault.end(),
+ [context](AVPixelFormat format) { return context->pix_fmt == format; });
+ auto pixelFormatFullRange = std::find_if(
+ kPixelFormatsFullRange.begin(), kPixelFormatsFullRange.end(),
+ [context](AVPixelFormat format) { return context->pix_fmt == format; });
+
// Limited or full range YUV420 is expected.
- RTC_CHECK(context->pix_fmt == kPixelFormatDefault ||
- context->pix_fmt == kPixelFormatFullRange);
+ RTC_CHECK(pixelFormatDefault != kPixelFormatsDefault.end() ||
+ pixelFormatFullRange != kPixelFormatsFullRange.end());
// `av_frame->width` and `av_frame->height` are set by FFmpeg. These are the
// actual image's dimensions and may be different from `context->width` and
@@ -112,8 +122,43 @@
// http://crbug.com/390941. Our pool is set up to zero-initialize new buffers.
// TODO(nisse): Delete that feature from the video pool, instead add
// an explicit call to InitializeData here.
- rtc::scoped_refptr<I420Buffer> frame_buffer =
- decoder->ffmpeg_buffer_pool_.CreateI420Buffer(width, height);
+ rtc::scoped_refptr<PlanarYuv8Buffer> frame_buffer;
+ rtc::scoped_refptr<I444Buffer> i444_buffer;
+ rtc::scoped_refptr<I420Buffer> i420_buffer;
+ switch (context->pix_fmt) {
+ case AV_PIX_FMT_YUV420P:
+ case AV_PIX_FMT_YUVJ420P:
+ i420_buffer =
+ decoder->ffmpeg_buffer_pool_.CreateI420Buffer(width, height);
+ // Set `av_frame` members as required by FFmpeg.
+ av_frame->data[kYPlaneIndex] = i420_buffer->MutableDataY();
+ av_frame->linesize[kYPlaneIndex] = i420_buffer->StrideY();
+ av_frame->data[kUPlaneIndex] = i420_buffer->MutableDataU();
+ av_frame->linesize[kUPlaneIndex] = i420_buffer->StrideU();
+ av_frame->data[kVPlaneIndex] = i420_buffer->MutableDataV();
+ av_frame->linesize[kVPlaneIndex] = i420_buffer->StrideV();
+ RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
+ frame_buffer = i420_buffer;
+ break;
+ case AV_PIX_FMT_YUV444P:
+ case AV_PIX_FMT_YUVJ444P:
+ i444_buffer =
+ decoder->ffmpeg_buffer_pool_.CreateI444Buffer(width, height);
+ // Set `av_frame` members as required by FFmpeg.
+ av_frame->data[kYPlaneIndex] = i444_buffer->MutableDataY();
+ av_frame->linesize[kYPlaneIndex] = i444_buffer->StrideY();
+ av_frame->data[kUPlaneIndex] = i444_buffer->MutableDataU();
+ av_frame->linesize[kUPlaneIndex] = i444_buffer->StrideU();
+ av_frame->data[kVPlaneIndex] = i444_buffer->MutableDataV();
+ av_frame->linesize[kVPlaneIndex] = i444_buffer->StrideV();
+ frame_buffer = i444_buffer;
+ break;
+ default:
+ RTC_LOG(LS_ERROR) << "Unsupported buffer type " << context->pix_fmt
+ << ". Check supported supported pixel formats!";
+ decoder->ReportError();
+ return -1;
+ }
int y_size = width * height;
int uv_size = frame_buffer->ChromaWidth() * frame_buffer->ChromaHeight();
@@ -125,15 +170,6 @@
av_frame->format = context->pix_fmt;
av_frame->reordered_opaque = context->reordered_opaque;
- // Set `av_frame` members as required by FFmpeg.
- av_frame->data[kYPlaneIndex] = frame_buffer->MutableDataY();
- av_frame->linesize[kYPlaneIndex] = frame_buffer->StrideY();
- av_frame->data[kUPlaneIndex] = frame_buffer->MutableDataU();
- av_frame->linesize[kUPlaneIndex] = frame_buffer->StrideU();
- av_frame->data[kVPlaneIndex] = frame_buffer->MutableDataV();
- av_frame->linesize[kVPlaneIndex] = frame_buffer->StrideV();
- RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
-
// Create a VideoFrame object, to keep a reference to the buffer.
// TODO(nisse): The VideoFrame's timestamp and rotation info is not used.
// Refactor to do not use a VideoFrame object at all.
@@ -197,7 +233,6 @@
av_context_->coded_width = resolution.Width();
av_context_->coded_height = resolution.Height();
}
- av_context_->pix_fmt = kPixelFormatDefault;
av_context_->extradata = nullptr;
av_context_->extradata_size = 0;
@@ -317,47 +352,103 @@
RTC_DCHECK(input_frame);
rtc::scoped_refptr<VideoFrameBuffer> frame_buffer =
input_frame->video_frame_buffer();
- const webrtc::I420BufferInterface* i420_buffer = frame_buffer->GetI420();
+
+ // Instantiate Planar YUV8 buffer according to video frame buffer type
+ const webrtc::PlanarYuv8Buffer* planar_yuv8_buffer = nullptr;
+ VideoFrameBuffer::Type video_frame_buffer_type = frame_buffer->type();
+ switch (video_frame_buffer_type) {
+ case VideoFrameBuffer::Type::kI420:
+ planar_yuv8_buffer = frame_buffer->GetI420();
+ break;
+ case VideoFrameBuffer::Type::kI444:
+ planar_yuv8_buffer = frame_buffer->GetI444();
+ break;
+ default:
+ // If this code is changed to allow other video frame buffer type,
+ // make sure that the code below which wraps I420/I444 buffer and
+ // code which converts to NV12 is changed
+ // to work with new video frame buffer type
+
+ RTC_LOG(LS_ERROR) << "frame_buffer type: "
+ << static_cast<int32_t>(video_frame_buffer_type)
+ << " is not supported!";
+ ReportError();
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
// When needed, FFmpeg applies cropping by moving plane pointers and adjusting
// frame width/height. Ensure that cropped buffers lie within the allocated
// memory.
- RTC_DCHECK_LE(av_frame_->width, i420_buffer->width());
- RTC_DCHECK_LE(av_frame_->height, i420_buffer->height());
- RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], i420_buffer->DataY());
- RTC_DCHECK_LE(
- av_frame_->data[kYPlaneIndex] +
- av_frame_->linesize[kYPlaneIndex] * av_frame_->height,
- i420_buffer->DataY() + i420_buffer->StrideY() * i420_buffer->height());
- RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], i420_buffer->DataU());
+ RTC_DCHECK_LE(av_frame_->width, planar_yuv8_buffer->width());
+ RTC_DCHECK_LE(av_frame_->height, planar_yuv8_buffer->height());
+ RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], planar_yuv8_buffer->DataY());
+ RTC_DCHECK_LE(av_frame_->data[kYPlaneIndex] +
+ av_frame_->linesize[kYPlaneIndex] * av_frame_->height,
+ planar_yuv8_buffer->DataY() + planar_yuv8_buffer->StrideY() *
+ planar_yuv8_buffer->height());
+ RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], planar_yuv8_buffer->DataU());
RTC_DCHECK_LE(av_frame_->data[kUPlaneIndex] +
av_frame_->linesize[kUPlaneIndex] * av_frame_->height / 2,
- i420_buffer->DataU() +
- i420_buffer->StrideU() * i420_buffer->height() / 2);
- RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], i420_buffer->DataV());
+ planar_yuv8_buffer->DataU() + planar_yuv8_buffer->StrideU() *
+ planar_yuv8_buffer->height() /
+ 2);
+ RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], planar_yuv8_buffer->DataV());
RTC_DCHECK_LE(av_frame_->data[kVPlaneIndex] +
av_frame_->linesize[kVPlaneIndex] * av_frame_->height / 2,
- i420_buffer->DataV() +
- i420_buffer->StrideV() * i420_buffer->height() / 2);
+ planar_yuv8_buffer->DataV() + planar_yuv8_buffer->StrideV() *
+ planar_yuv8_buffer->height() /
+ 2);
- rtc::scoped_refptr<webrtc::VideoFrameBuffer> cropped_buffer = WrapI420Buffer(
- av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
- av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
- av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
- av_frame_->linesize[kVPlaneIndex],
- // To keep reference alive.
- [frame_buffer] {});
+ rtc::scoped_refptr<webrtc::VideoFrameBuffer> cropped_buffer;
+ if (video_frame_buffer_type == VideoFrameBuffer::Type::kI420) {
+ cropped_buffer = WrapI420Buffer(
+ av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
+ av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
+ av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
+ av_frame_->linesize[kVPlaneIndex],
+ // To keep reference alive.
+ [frame_buffer] {});
+ } else {
+ cropped_buffer = WrapI444Buffer(
+ av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
+ av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
+ av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
+ av_frame_->linesize[kVPlaneIndex],
+ // To keep reference alive.
+ [frame_buffer] {});
+ }
if (preferred_output_format_ == VideoFrameBuffer::Type::kNV12) {
- const I420BufferInterface* cropped_i420 = cropped_buffer->GetI420();
auto nv12_buffer = output_buffer_pool_.CreateNV12Buffer(
- cropped_i420->width(), cropped_i420->height());
- libyuv::I420ToNV12(cropped_i420->DataY(), cropped_i420->StrideY(),
- cropped_i420->DataU(), cropped_i420->StrideU(),
- cropped_i420->DataV(), cropped_i420->StrideV(),
- nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
- nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
- i420_buffer->width(), i420_buffer->height());
+ cropped_buffer->width(), cropped_buffer->height());
+
+ const PlanarYuv8Buffer* cropped_planar_yuv8_buffer = nullptr;
+ if (video_frame_buffer_type == VideoFrameBuffer::Type::kI420) {
+ cropped_planar_yuv8_buffer = cropped_buffer->GetI420();
+ libyuv::I420ToNV12(cropped_planar_yuv8_buffer->DataY(),
+ cropped_planar_yuv8_buffer->StrideY(),
+ cropped_planar_yuv8_buffer->DataU(),
+ cropped_planar_yuv8_buffer->StrideU(),
+ cropped_planar_yuv8_buffer->DataV(),
+ cropped_planar_yuv8_buffer->StrideV(),
+ nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
+ nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
+ planar_yuv8_buffer->width(),
+ planar_yuv8_buffer->height());
+ } else {
+ cropped_planar_yuv8_buffer = cropped_buffer->GetI444();
+ libyuv::I444ToNV12(cropped_planar_yuv8_buffer->DataY(),
+ cropped_planar_yuv8_buffer->StrideY(),
+ cropped_planar_yuv8_buffer->DataU(),
+ cropped_planar_yuv8_buffer->StrideU(),
+ cropped_planar_yuv8_buffer->DataV(),
+ cropped_planar_yuv8_buffer->StrideV(),
+ nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
+ nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
+ planar_yuv8_buffer->width(),
+ planar_yuv8_buffer->height());
+ }
+
cropped_buffer = nv12_buffer;
}
diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm
index 966eb3e..93ed03a 100644
--- a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm
+++ b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm
@@ -240,6 +240,7 @@
case webrtc::H264Profile::kProfileConstrainedHigh:
case webrtc::H264Profile::kProfileHigh:
+ case webrtc::H264Profile::kProfilePredictiveHigh444:
switch (profile_level_id.level) {
case webrtc::H264Level::kLevel3:
return kVTProfileLevel_H264_High_3_0;