Added support for H264 YUV444 (I444) decoding.
Added Nutanix Inc. to the AUTHORS file.
PS#1 is a reland of "Added support for H264 YUV444 (I444) decoding." https://webrtc-review.googlesource.com/c/src/+/234540
Bug: chromium:1251096
Change-Id: I99a1b1e4d8b60192ff96f92334a430240875c66c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/235340
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Harald Alvestrand <hta@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35684}
diff --git a/AUTHORS b/AUTHORS
index eb650c5..e4729a5 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -138,6 +138,7 @@
MIPS Technologies <*@mips.com>
Mozilla Foundation <*@mozilla.com>
Netgem S.A. <*@netgem.com>
+Nutanix Inc. <*@nutanix.com>
NVIDIA Corporation <*@nvidia.com>
Opera Software ASA <*@opera.com>
Optical Tone Ltd <*@opticaltone.com>
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index e6052fe..9fd2891 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -43,6 +43,8 @@
sources = [
"i420_buffer.cc",
"i420_buffer.h",
+ "i444_buffer.cc",
+ "i444_buffer.h",
"nv12_buffer.cc",
"nv12_buffer.h",
"video_codec_type.h",
diff --git a/api/video/DEPS b/api/video/DEPS
index cf6770d..5a3e496 100644
--- a/api/video/DEPS
+++ b/api/video/DEPS
@@ -18,6 +18,10 @@
"+rtc_base/memory/aligned_malloc.h",
],
+ "i444_buffer\.h": [
+ "+rtc_base/memory/aligned_malloc.h",
+ ],
+
"nv12_buffer\.h": [
"+rtc_base/memory/aligned_malloc.h",
],
diff --git a/api/video/i444_buffer.cc b/api/video/i444_buffer.cc
new file mode 100644
index 0000000..5f8b714
--- /dev/null
+++ b/api/video/i444_buffer.cc
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/video/i444_buffer.h"
+
+#include <string.h>
+
+#include <algorithm>
+#include <utility>
+
+#include "api/video/i420_buffer.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/ref_counted_object.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+#include "third_party/libyuv/include/libyuv/planar_functions.h"
+#include "third_party/libyuv/include/libyuv/scale.h"
+
+// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD.
+static const int kBufferAlignment = 64;
+
+namespace webrtc {
+
+namespace {
+
+int I444DataSize(int height, int stride_y, int stride_u, int stride_v) {
+ return stride_y * height + stride_u * height + stride_v * height;
+}
+
+} // namespace
+
+I444Buffer::I444Buffer(int width, int height)
+ : I444Buffer(width, height, width, (width), (width)) {}
+
+I444Buffer::I444Buffer(int width,
+ int height,
+ int stride_y,
+ int stride_u,
+ int stride_v)
+ : width_(width),
+ height_(height),
+ stride_y_(stride_y),
+ stride_u_(stride_u),
+ stride_v_(stride_v),
+ data_(static_cast<uint8_t*>(
+ AlignedMalloc(I444DataSize(height, stride_y, stride_u, stride_v),
+ kBufferAlignment))) {
+ RTC_DCHECK_GT(width, 0);
+ RTC_DCHECK_GT(height, 0);
+ RTC_DCHECK_GE(stride_y, width);
+ RTC_DCHECK_GE(stride_u, (width));
+ RTC_DCHECK_GE(stride_v, (width));
+}
+
+I444Buffer::~I444Buffer() {}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Create(int width, int height) {
+ return new rtc::RefCountedObject<I444Buffer>(width, height);
+}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Create(int width,
+ int height,
+ int stride_y,
+ int stride_u,
+ int stride_v) {
+ return new rtc::RefCountedObject<I444Buffer>(width, height, stride_y,
+ stride_u, stride_v);
+}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Copy(
+ const I444BufferInterface& source) {
+ return Copy(source.width(), source.height(), source.DataY(), source.StrideY(),
+ source.DataU(), source.StrideU(), source.DataV(),
+ source.StrideV());
+}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Copy(int width,
+ int height,
+ const uint8_t* data_y,
+ int stride_y,
+ const uint8_t* data_u,
+ int stride_u,
+ const uint8_t* data_v,
+ int stride_v) {
+ // Note: May use different strides than the input data.
+ rtc::scoped_refptr<I444Buffer> buffer = Create(width, height);
+ RTC_CHECK_EQ(0, libyuv::I444Copy(data_y, stride_y, data_u, stride_u, data_v,
+ stride_v, buffer->MutableDataY(),
+ buffer->StrideY(), buffer->MutableDataU(),
+ buffer->StrideU(), buffer->MutableDataV(),
+ buffer->StrideV(), width, height));
+ return buffer;
+}
+
+// static
+rtc::scoped_refptr<I444Buffer> I444Buffer::Rotate(
+ const I444BufferInterface& src,
+ VideoRotation rotation) {
+ RTC_CHECK(src.DataY());
+ RTC_CHECK(src.DataU());
+ RTC_CHECK(src.DataV());
+
+ int rotated_width = src.width();
+ int rotated_height = src.height();
+ if (rotation == webrtc::kVideoRotation_90 ||
+ rotation == webrtc::kVideoRotation_270) {
+ std::swap(rotated_width, rotated_height);
+ }
+
+ rtc::scoped_refptr<webrtc::I444Buffer> buffer =
+ I444Buffer::Create(rotated_width, rotated_height);
+
+ RTC_CHECK_EQ(0,
+ libyuv::I444Rotate(
+ src.DataY(), src.StrideY(), src.DataU(), src.StrideU(),
+ src.DataV(), src.StrideV(), buffer->MutableDataY(),
+ buffer->StrideY(), buffer->MutableDataU(), buffer->StrideU(),
+ buffer->MutableDataV(), buffer->StrideV(), src.width(),
+ src.height(), static_cast<libyuv::RotationMode>(rotation)));
+
+ return buffer;
+}
+
+rtc::scoped_refptr<I420BufferInterface> I444Buffer::ToI420() {
+ rtc::scoped_refptr<I420Buffer> i420_buffer =
+ I420Buffer::Create(width(), height());
+ libyuv::I444ToI420(DataY(), StrideY(), DataU(), StrideU(), DataV(), StrideV(),
+ i420_buffer->MutableDataY(), i420_buffer->StrideY(),
+ i420_buffer->MutableDataU(), i420_buffer->StrideU(),
+ i420_buffer->MutableDataV(), i420_buffer->StrideV(),
+ width(), height());
+ return i420_buffer;
+}
+
+void I444Buffer::InitializeData() {
+ memset(data_.get(), 0,
+ I444DataSize(height_, stride_y_, stride_u_, stride_v_));
+}
+
+int I444Buffer::width() const {
+ return width_;
+}
+
+int I444Buffer::height() const {
+ return height_;
+}
+
+const uint8_t* I444Buffer::DataY() const {
+ return data_.get();
+}
+const uint8_t* I444Buffer::DataU() const {
+ return data_.get() + stride_y_ * height_;
+}
+const uint8_t* I444Buffer::DataV() const {
+ return data_.get() + stride_y_ * height_ + stride_u_ * ((height_));
+}
+
+int I444Buffer::StrideY() const {
+ return stride_y_;
+}
+int I444Buffer::StrideU() const {
+ return stride_u_;
+}
+int I444Buffer::StrideV() const {
+ return stride_v_;
+}
+
+uint8_t* I444Buffer::MutableDataY() {
+ return const_cast<uint8_t*>(DataY());
+}
+uint8_t* I444Buffer::MutableDataU() {
+ return const_cast<uint8_t*>(DataU());
+}
+uint8_t* I444Buffer::MutableDataV() {
+ return const_cast<uint8_t*>(DataV());
+}
+
+void I444Buffer::CropAndScaleFrom(const I444BufferInterface& src,
+ int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height) {
+ RTC_CHECK_LE(crop_width, src.width());
+ RTC_CHECK_LE(crop_height, src.height());
+ RTC_CHECK_LE(crop_width + offset_x, src.width());
+ RTC_CHECK_LE(crop_height + offset_y, src.height());
+ RTC_CHECK_GE(offset_x, 0);
+ RTC_CHECK_GE(offset_y, 0);
+
+ const uint8_t* y_plane = src.DataY() + src.StrideY() * offset_y + offset_x;
+ const uint8_t* u_plane = src.DataU() + src.StrideU() * offset_y + offset_x;
+ const uint8_t* v_plane = src.DataV() + src.StrideV() * offset_y + offset_x;
+ int res =
+ libyuv::I444Scale(y_plane, src.StrideY(), u_plane, src.StrideU(), v_plane,
+ src.StrideV(), crop_width, crop_height, MutableDataY(),
+ StrideY(), MutableDataU(), StrideU(), MutableDataV(),
+ StrideV(), width(), height(), libyuv::kFilterBox);
+
+ RTC_DCHECK_EQ(res, 0);
+}
+
+} // namespace webrtc
diff --git a/api/video/i444_buffer.h b/api/video/i444_buffer.h
new file mode 100644
index 0000000..d46c245
--- /dev/null
+++ b/api/video/i444_buffer.h
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_I444_BUFFER_H_
+#define API_VIDEO_I444_BUFFER_H_
+
+#include <stdint.h>
+
+#include <memory>
+
+#include "api/scoped_refptr.h"
+#include "api/video/video_frame_buffer.h"
+#include "api/video/video_rotation.h"
+#include "rtc_base/memory/aligned_malloc.h"
+#include "rtc_base/system/rtc_export.h"
+
+namespace webrtc {
+
+// Plain I444 buffer in standard memory.
+// I444 represents an image with in YUV format withouth any chroma subsampling.
+// https://en.wikipedia.org/wiki/Chroma_subsampling#4:4:4
+class RTC_EXPORT I444Buffer : public I444BufferInterface {
+ public:
+ static rtc::scoped_refptr<I444Buffer> Create(int width, int height);
+ static rtc::scoped_refptr<I444Buffer> Create(int width,
+ int height,
+ int stride_y,
+ int stride_u,
+ int stride_v);
+
+ // Create a new buffer and copy the pixel data.
+ static rtc::scoped_refptr<I444Buffer> Copy(const I444BufferInterface& buffer);
+
+ static rtc::scoped_refptr<I444Buffer> Copy(int width,
+ int height,
+ const uint8_t* data_y,
+ int stride_y,
+ const uint8_t* data_u,
+ int stride_u,
+ const uint8_t* data_v,
+ int stride_v);
+
+ // Returns a rotated copy of |src|.
+ static rtc::scoped_refptr<I444Buffer> Rotate(const I444BufferInterface& src,
+ VideoRotation rotation);
+
+ rtc::scoped_refptr<I420BufferInterface> ToI420() final;
+ const I420BufferInterface* GetI420() const final { return nullptr; }
+
+ // Sets all three planes to all zeros. Used to work around for
+ // quirks in memory checkers
+ // (https://bugs.chromium.org/p/libyuv/issues/detail?id=377) and
+ // ffmpeg (http://crbug.com/390941).
+ // TODO(nisse): Deprecated. Should be deleted if/when those issues
+ // are resolved in a better way. Or in the mean time, use SetBlack.
+ void InitializeData();
+
+ int width() const override;
+ int height() const override;
+ const uint8_t* DataY() const override;
+ const uint8_t* DataU() const override;
+ const uint8_t* DataV() const override;
+
+ int StrideY() const override;
+ int StrideU() const override;
+ int StrideV() const override;
+
+ uint8_t* MutableDataY();
+ uint8_t* MutableDataU();
+ uint8_t* MutableDataV();
+
+ // Scale the cropped area of |src| to the size of |this| buffer, and
+ // write the result into |this|.
+ void CropAndScaleFrom(const I444BufferInterface& src,
+ int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height);
+
+ protected:
+ I444Buffer(int width, int height);
+ I444Buffer(int width, int height, int stride_y, int stride_u, int stride_v);
+
+ ~I444Buffer() override;
+
+ private:
+ const int width_;
+ const int height_;
+ const int stride_y_;
+ const int stride_u_;
+ const int stride_v_;
+ const std::unique_ptr<uint8_t, AlignedFreeDeleter> data_;
+};
+
+} // namespace webrtc
+
+#endif // API_VIDEO_I444_BUFFER_H_
diff --git a/api/video/test/BUILD.gn b/api/video/test/BUILD.gn
index 1573e78..5b0d57b 100644
--- a/api/video/test/BUILD.gn
+++ b/api/video/test/BUILD.gn
@@ -12,6 +12,7 @@
testonly = true
sources = [
"color_space_unittest.cc",
+ "i444_buffer_unittest.cc",
"nv12_buffer_unittest.cc",
"video_adaptation_counters_unittest.cc",
"video_bitrate_allocation_unittest.cc",
diff --git a/api/video/test/i444_buffer_unittest.cc b/api/video/test/i444_buffer_unittest.cc
new file mode 100644
index 0000000..52ce49a
--- /dev/null
+++ b/api/video/test/i444_buffer_unittest.cc
@@ -0,0 +1,112 @@
+
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/i444_buffer.h"
+
+#include "api/video/i420_buffer.h"
+#include "test/frame_utils.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+namespace {
+int GetY(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
+ return buf->DataY()[row * buf->StrideY() + col];
+}
+
+int GetU(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
+ return buf->DataU()[row * buf->StrideU() + col];
+}
+
+int GetV(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
+ return buf->DataV()[row * buf->StrideV() + col];
+}
+
+void FillI444Buffer(rtc::scoped_refptr<I444Buffer> buf) {
+ const uint8_t Y = 1;
+ const uint8_t U = 2;
+ const uint8_t V = 3;
+ for (int row = 0; row < buf->height(); ++row) {
+ for (int col = 0; col < buf->width(); ++col) {
+ buf->MutableDataY()[row * buf->StrideY() + col] = Y;
+ buf->MutableDataU()[row * buf->StrideU() + col] = U;
+ buf->MutableDataV()[row * buf->StrideV() + col] = V;
+ }
+ }
+}
+
+} // namespace
+
+TEST(I444BufferTest, InitialData) {
+ constexpr int stride = 3;
+ constexpr int width = 3;
+ constexpr int height = 3;
+
+ rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
+ EXPECT_EQ(width, i444_buffer->width());
+ EXPECT_EQ(height, i444_buffer->height());
+ EXPECT_EQ(stride, i444_buffer->StrideY());
+ EXPECT_EQ(stride, i444_buffer->StrideU());
+ EXPECT_EQ(stride, i444_buffer->StrideV());
+ EXPECT_EQ(3, i444_buffer->ChromaWidth());
+ EXPECT_EQ(3, i444_buffer->ChromaHeight());
+}
+
+TEST(I444BufferTest, ReadPixels) {
+ constexpr int width = 3;
+ constexpr int height = 3;
+
+ rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
+ // Y = 1, U = 2, V = 3.
+ FillI444Buffer(i444_buffer);
+ for (int row = 0; row < height; row++) {
+ for (int col = 0; col < width; col++) {
+ EXPECT_EQ(1, GetY(i444_buffer, col, row));
+ EXPECT_EQ(2, GetU(i444_buffer, col, row));
+ EXPECT_EQ(3, GetV(i444_buffer, col, row));
+ }
+ }
+}
+
+TEST(I444BufferTest, ToI420) {
+ constexpr int width = 3;
+ constexpr int height = 3;
+ constexpr int size_y = width * height;
+ constexpr int size_u = (width + 1) / 2 * (height + 1) / 2;
+ constexpr int size_v = (width + 1) / 2 * (height + 1) / 2;
+ rtc::scoped_refptr<I420Buffer> reference(I420Buffer::Create(width, height));
+ memset(reference->MutableDataY(), 8, size_y);
+ memset(reference->MutableDataU(), 4, size_u);
+ memset(reference->MutableDataV(), 2, size_v);
+
+ rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
+ // Convert the reference buffer to I444.
+ memset(i444_buffer->MutableDataY(), 8, size_y);
+ memset(i444_buffer->MutableDataU(), 4, size_y);
+ memset(i444_buffer->MutableDataV(), 2, size_y);
+
+ // Confirm YUV values are as expected.
+ for (int row = 0; row < height; row++) {
+ for (int col = 0; col < width; col++) {
+ EXPECT_EQ(8, GetY(i444_buffer, col, row));
+ EXPECT_EQ(4, GetU(i444_buffer, col, row));
+ EXPECT_EQ(2, GetV(i444_buffer, col, row));
+ }
+ }
+
+ rtc::scoped_refptr<I420BufferInterface> i420_buffer(i444_buffer->ToI420());
+ EXPECT_EQ(height, i420_buffer->height());
+ EXPECT_EQ(width, i420_buffer->width());
+ EXPECT_TRUE(test::FrameBufsEqual(reference, i420_buffer));
+}
+
+} // namespace webrtc
diff --git a/api/video/video_frame_buffer.cc b/api/video/video_frame_buffer.cc
index f6904b5..6c46f78 100644
--- a/api/video/video_frame_buffer.cc
+++ b/api/video/video_frame_buffer.cc
@@ -11,6 +11,7 @@
#include "api/video/video_frame_buffer.h"
#include "api/video/i420_buffer.h"
+#include "api/video/i444_buffer.h"
#include "api/video/nv12_buffer.h"
#include "rtc_base/checks.h"
@@ -117,6 +118,19 @@
return height();
}
+rtc::scoped_refptr<VideoFrameBuffer> I444BufferInterface::CropAndScale(
+ int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height,
+ int scaled_width,
+ int scaled_height) {
+ rtc::scoped_refptr<I444Buffer> result =
+ I444Buffer::Create(scaled_width, scaled_height);
+ result->CropAndScaleFrom(*this, offset_x, offset_y, crop_width, crop_height);
+ return result;
+}
+
VideoFrameBuffer::Type I010BufferInterface::type() const {
return Type::kI010;
}
diff --git a/api/video/video_frame_buffer.h b/api/video/video_frame_buffer.h
index 7b0782f..6098a481 100644
--- a/api/video/video_frame_buffer.h
+++ b/api/video/video_frame_buffer.h
@@ -184,6 +184,13 @@
int ChromaWidth() const final;
int ChromaHeight() const final;
+ rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height,
+ int scaled_width,
+ int scaled_height) override;
+
protected:
~I444BufferInterface() override {}
};
diff --git a/api/video_codecs/h264_profile_level_id.cc b/api/video_codecs/h264_profile_level_id.cc
index fa47758..02b43ba 100644
--- a/api/video_codecs/h264_profile_level_id.cc
+++ b/api/video_codecs/h264_profile_level_id.cc
@@ -68,7 +68,8 @@
{0x58, BitPattern("10xx0000"), H264Profile::kProfileBaseline},
{0x4D, BitPattern("0x0x0000"), H264Profile::kProfileMain},
{0x64, BitPattern("00000000"), H264Profile::kProfileHigh},
- {0x64, BitPattern("00001100"), H264Profile::kProfileConstrainedHigh}};
+ {0x64, BitPattern("00001100"), H264Profile::kProfileConstrainedHigh},
+ {0xF4, BitPattern("00000000"), H264Profile::kProfilePredictiveHigh444}};
struct LevelConstraint {
const int max_macroblocks_per_second;
@@ -228,6 +229,9 @@
case H264Profile::kProfileHigh:
profile_idc_iop_string = "6400";
break;
+ case H264Profile::kProfilePredictiveHigh444:
+ profile_idc_iop_string = "f400";
+ break;
// Unrecognized profile.
default:
return absl::nullopt;
diff --git a/api/video_codecs/h264_profile_level_id.h b/api/video_codecs/h264_profile_level_id.h
index 51d025c..4b46ad3 100644
--- a/api/video_codecs/h264_profile_level_id.h
+++ b/api/video_codecs/h264_profile_level_id.h
@@ -25,6 +25,7 @@
kProfileMain,
kProfileConstrainedHigh,
kProfileHigh,
+ kProfilePredictiveHigh444,
};
// All values are equal to ten times the level number, except level 1b which is
diff --git a/common_video/include/video_frame_buffer_pool.h b/common_video/include/video_frame_buffer_pool.h
index 539a6cc..f26a9f7 100644
--- a/common_video/include/video_frame_buffer_pool.h
+++ b/common_video/include/video_frame_buffer_pool.h
@@ -17,6 +17,7 @@
#include "api/scoped_refptr.h"
#include "api/video/i420_buffer.h"
+#include "api/video/i444_buffer.h"
#include "api/video/nv12_buffer.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/ref_counted_object.h"
@@ -43,6 +44,7 @@
// and there are less than `max_number_of_buffers` pending, a buffer is
// created. Returns null otherwise.
rtc::scoped_refptr<I420Buffer> CreateI420Buffer(int width, int height);
+ rtc::scoped_refptr<I444Buffer> CreateI444Buffer(int width, int height);
rtc::scoped_refptr<NV12Buffer> CreateNV12Buffer(int width, int height);
// Changes the max amount of buffers in the pool to the new value.
diff --git a/common_video/video_frame_buffer_pool.cc b/common_video/video_frame_buffer_pool.cc
index 9c88f0b..e95eac3 100644
--- a/common_video/video_frame_buffer_pool.cc
+++ b/common_video/video_frame_buffer_pool.cc
@@ -20,12 +20,17 @@
bool HasOneRef(const rtc::scoped_refptr<VideoFrameBuffer>& buffer) {
// Cast to rtc::RefCountedObject is safe because this function is only called
// on locally created VideoFrameBuffers, which are either
- // `rtc::RefCountedObject<I420Buffer>` or `rtc::RefCountedObject<NV12Buffer>`.
+ // `rtc::RefCountedObject<I420Buffer>`, `rtc::RefCountedObject<I444Buffer>` or
+ // `rtc::RefCountedObject<NV12Buffer>`.
switch (buffer->type()) {
case VideoFrameBuffer::Type::kI420: {
return static_cast<rtc::RefCountedObject<I420Buffer>*>(buffer.get())
->HasOneRef();
}
+ case VideoFrameBuffer::Type::kI444: {
+ return static_cast<rtc::RefCountedObject<I444Buffer>*>(buffer.get())
+ ->HasOneRef();
+ }
case VideoFrameBuffer::Type::kNV12: {
return static_cast<rtc::RefCountedObject<NV12Buffer>*>(buffer.get())
->HasOneRef();
@@ -116,6 +121,37 @@
return buffer;
}
+rtc::scoped_refptr<I444Buffer> VideoFrameBufferPool::CreateI444Buffer(
+ int width,
+ int height) {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
+
+ rtc::scoped_refptr<VideoFrameBuffer> existing_buffer =
+ GetExistingBuffer(width, height, VideoFrameBuffer::Type::kI444);
+ if (existing_buffer) {
+ // Cast is safe because the only way kI444 buffer is created is
+ // in the same function below, where |RefCountedObject<I444Buffer>|
+ // is created.
+ rtc::RefCountedObject<I444Buffer>* raw_buffer =
+ static_cast<rtc::RefCountedObject<I444Buffer>*>(existing_buffer.get());
+ // Creates a new scoped_refptr, which is also pointing to the same
+ // RefCountedObject as buffer, increasing ref count.
+ return rtc::scoped_refptr<I444Buffer>(raw_buffer);
+ }
+
+ if (buffers_.size() >= max_number_of_buffers_)
+ return nullptr;
+ // Allocate new buffer.
+ rtc::scoped_refptr<I444Buffer> buffer =
+ new rtc::RefCountedObject<I444Buffer>(width, height);
+
+ if (zero_initialize_)
+ buffer->InitializeData();
+
+ buffers_.push_back(buffer);
+ return buffer;
+}
+
rtc::scoped_refptr<NV12Buffer> VideoFrameBufferPool::CreateNV12Buffer(
int width,
int height) {
diff --git a/media/engine/internal_decoder_factory.cc b/media/engine/internal_decoder_factory.cc
index faac91e..a63d05e 100644
--- a/media/engine/internal_decoder_factory.cc
+++ b/media/engine/internal_decoder_factory.cc
@@ -47,7 +47,7 @@
formats.push_back(SdpVideoFormat(cricket::kVp8CodecName));
for (const SdpVideoFormat& format : SupportedVP9DecoderCodecs())
formats.push_back(format);
- for (const SdpVideoFormat& h264_format : SupportedH264Codecs())
+ for (const SdpVideoFormat& h264_format : SupportedH264DecoderCodecs())
formats.push_back(h264_format);
if (kIsLibaomAv1DecoderSupported ||
diff --git a/modules/video_coding/codecs/h264/h264.cc b/modules/video_coding/codecs/h264/h264.cc
index 8324b7c..2ac19ba 100644
--- a/modules/video_coding/codecs/h264/h264.cc
+++ b/modules/video_coding/codecs/h264/h264.cc
@@ -80,18 +80,34 @@
//
// We support both packetization modes 0 (mandatory) and 1 (optional,
// preferred).
- return {CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1,
- "1"),
- CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1,
- "0"),
- CreateH264Format(H264Profile::kProfileConstrainedBaseline,
- H264Level::kLevel3_1, "1"),
- CreateH264Format(H264Profile::kProfileConstrainedBaseline,
- H264Level::kLevel3_1, "0"),
- CreateH264Format(H264Profile::kProfileMain,
- H264Level::kLevel3_1, "1"),
- CreateH264Format(H264Profile::kProfileMain,
- H264Level::kLevel3_1, "0")};
+ return {
+ CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1,
+ "1"),
+ CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1,
+ "0"),
+ CreateH264Format(H264Profile::kProfileConstrainedBaseline,
+ H264Level::kLevel3_1, "1"),
+ CreateH264Format(H264Profile::kProfileConstrainedBaseline,
+ H264Level::kLevel3_1, "0"),
+ CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "1"),
+ CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "0")};
+}
+
+std::vector<SdpVideoFormat> SupportedH264DecoderCodecs() {
+ TRACE_EVENT0("webrtc", __func__);
+ if (!IsH264CodecSupported())
+ return std::vector<SdpVideoFormat>();
+
+ std::vector<SdpVideoFormat> supportedCodecs = SupportedH264Codecs();
+
+ // OpenH264 doesn't yet support High Predictive 4:4:4 encoding but it does
+ // support decoding.
+ supportedCodecs.push_back(CreateH264Format(
+ H264Profile::kProfilePredictiveHigh444, H264Level::kLevel3_1, "1"));
+ supportedCodecs.push_back(CreateH264Format(
+ H264Profile::kProfilePredictiveHigh444, H264Level::kLevel3_1, "0"));
+
+ return supportedCodecs;
}
std::unique_ptr<H264Encoder> H264Encoder::Create(
diff --git a/modules/video_coding/codecs/h264/h264_decoder_impl.cc b/modules/video_coding/codecs/h264/h264_decoder_impl.cc
index b42aac5..31279b7 100644
--- a/modules/video_coding/codecs/h264/h264_decoder_impl.cc
+++ b/modules/video_coding/codecs/h264/h264_decoder_impl.cc
@@ -41,8 +41,10 @@
namespace {
-const AVPixelFormat kPixelFormatDefault = AV_PIX_FMT_YUV420P;
-const AVPixelFormat kPixelFormatFullRange = AV_PIX_FMT_YUVJ420P;
+constexpr std::array<AVPixelFormat, 2> kPixelFormatsDefault = {
+ AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV444P};
+constexpr std::array<AVPixelFormat, 2> kPixelFormatsFullRange = {
+ AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ444P};
const size_t kYPlaneIndex = 0;
const size_t kUPlaneIndex = 1;
const size_t kVPlaneIndex = 2;
@@ -76,9 +78,17 @@
// Necessary capability to be allowed to provide our own buffers.
RTC_DCHECK(context->codec->capabilities | AV_CODEC_CAP_DR1);
+ // Limited or full range YUV420 or YUV444 is expected.
+ auto pixelFormatDefault = std::find_if(
+ kPixelFormatsDefault.begin(), kPixelFormatsDefault.end(),
+ [context](AVPixelFormat format) { return context->pix_fmt == format; });
+ auto pixelFormatFullRange = std::find_if(
+ kPixelFormatsFullRange.begin(), kPixelFormatsFullRange.end(),
+ [context](AVPixelFormat format) { return context->pix_fmt == format; });
+
// Limited or full range YUV420 is expected.
- RTC_CHECK(context->pix_fmt == kPixelFormatDefault ||
- context->pix_fmt == kPixelFormatFullRange);
+ RTC_CHECK(pixelFormatDefault != kPixelFormatsDefault.end() ||
+ pixelFormatFullRange != kPixelFormatsFullRange.end());
// `av_frame->width` and `av_frame->height` are set by FFmpeg. These are the
// actual image's dimensions and may be different from `context->width` and
@@ -112,8 +122,43 @@
// http://crbug.com/390941. Our pool is set up to zero-initialize new buffers.
// TODO(nisse): Delete that feature from the video pool, instead add
// an explicit call to InitializeData here.
- rtc::scoped_refptr<I420Buffer> frame_buffer =
- decoder->ffmpeg_buffer_pool_.CreateI420Buffer(width, height);
+ rtc::scoped_refptr<PlanarYuv8Buffer> frame_buffer;
+ rtc::scoped_refptr<I444Buffer> i444_buffer;
+ rtc::scoped_refptr<I420Buffer> i420_buffer;
+ switch (context->pix_fmt) {
+ case AV_PIX_FMT_YUV420P:
+ case AV_PIX_FMT_YUVJ420P:
+ i420_buffer =
+ decoder->ffmpeg_buffer_pool_.CreateI420Buffer(width, height);
+ // Set `av_frame` members as required by FFmpeg.
+ av_frame->data[kYPlaneIndex] = i420_buffer->MutableDataY();
+ av_frame->linesize[kYPlaneIndex] = i420_buffer->StrideY();
+ av_frame->data[kUPlaneIndex] = i420_buffer->MutableDataU();
+ av_frame->linesize[kUPlaneIndex] = i420_buffer->StrideU();
+ av_frame->data[kVPlaneIndex] = i420_buffer->MutableDataV();
+ av_frame->linesize[kVPlaneIndex] = i420_buffer->StrideV();
+ RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
+ frame_buffer = i420_buffer;
+ break;
+ case AV_PIX_FMT_YUV444P:
+ case AV_PIX_FMT_YUVJ444P:
+ i444_buffer =
+ decoder->ffmpeg_buffer_pool_.CreateI444Buffer(width, height);
+ // Set `av_frame` members as required by FFmpeg.
+ av_frame->data[kYPlaneIndex] = i444_buffer->MutableDataY();
+ av_frame->linesize[kYPlaneIndex] = i444_buffer->StrideY();
+ av_frame->data[kUPlaneIndex] = i444_buffer->MutableDataU();
+ av_frame->linesize[kUPlaneIndex] = i444_buffer->StrideU();
+ av_frame->data[kVPlaneIndex] = i444_buffer->MutableDataV();
+ av_frame->linesize[kVPlaneIndex] = i444_buffer->StrideV();
+ frame_buffer = i444_buffer;
+ break;
+ default:
+ RTC_LOG(LS_ERROR) << "Unsupported buffer type " << context->pix_fmt
+ << ". Check supported supported pixel formats!";
+ decoder->ReportError();
+ return -1;
+ }
int y_size = width * height;
int uv_size = frame_buffer->ChromaWidth() * frame_buffer->ChromaHeight();
@@ -125,15 +170,6 @@
av_frame->format = context->pix_fmt;
av_frame->reordered_opaque = context->reordered_opaque;
- // Set `av_frame` members as required by FFmpeg.
- av_frame->data[kYPlaneIndex] = frame_buffer->MutableDataY();
- av_frame->linesize[kYPlaneIndex] = frame_buffer->StrideY();
- av_frame->data[kUPlaneIndex] = frame_buffer->MutableDataU();
- av_frame->linesize[kUPlaneIndex] = frame_buffer->StrideU();
- av_frame->data[kVPlaneIndex] = frame_buffer->MutableDataV();
- av_frame->linesize[kVPlaneIndex] = frame_buffer->StrideV();
- RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
-
// Create a VideoFrame object, to keep a reference to the buffer.
// TODO(nisse): The VideoFrame's timestamp and rotation info is not used.
// Refactor to do not use a VideoFrame object at all.
@@ -197,7 +233,6 @@
av_context_->coded_width = resolution.Width();
av_context_->coded_height = resolution.Height();
}
- av_context_->pix_fmt = kPixelFormatDefault;
av_context_->extradata = nullptr;
av_context_->extradata_size = 0;
@@ -317,47 +352,103 @@
RTC_DCHECK(input_frame);
rtc::scoped_refptr<VideoFrameBuffer> frame_buffer =
input_frame->video_frame_buffer();
- const webrtc::I420BufferInterface* i420_buffer = frame_buffer->GetI420();
+
+ // Instantiate Planar YUV8 buffer according to video frame buffer type
+ const webrtc::PlanarYuv8Buffer* planar_yuv8_buffer = nullptr;
+ VideoFrameBuffer::Type video_frame_buffer_type = frame_buffer->type();
+ switch (video_frame_buffer_type) {
+ case VideoFrameBuffer::Type::kI420:
+ planar_yuv8_buffer = frame_buffer->GetI420();
+ break;
+ case VideoFrameBuffer::Type::kI444:
+ planar_yuv8_buffer = frame_buffer->GetI444();
+ break;
+ default:
+ // If this code is changed to allow other video frame buffer type,
+ // make sure that the code below which wraps I420/I444 buffer and
+ // code which converts to NV12 is changed
+ // to work with new video frame buffer type
+
+ RTC_LOG(LS_ERROR) << "frame_buffer type: "
+ << static_cast<int32_t>(video_frame_buffer_type)
+ << " is not supported!";
+ ReportError();
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
// When needed, FFmpeg applies cropping by moving plane pointers and adjusting
// frame width/height. Ensure that cropped buffers lie within the allocated
// memory.
- RTC_DCHECK_LE(av_frame_->width, i420_buffer->width());
- RTC_DCHECK_LE(av_frame_->height, i420_buffer->height());
- RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], i420_buffer->DataY());
- RTC_DCHECK_LE(
- av_frame_->data[kYPlaneIndex] +
- av_frame_->linesize[kYPlaneIndex] * av_frame_->height,
- i420_buffer->DataY() + i420_buffer->StrideY() * i420_buffer->height());
- RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], i420_buffer->DataU());
+ RTC_DCHECK_LE(av_frame_->width, planar_yuv8_buffer->width());
+ RTC_DCHECK_LE(av_frame_->height, planar_yuv8_buffer->height());
+ RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], planar_yuv8_buffer->DataY());
+ RTC_DCHECK_LE(av_frame_->data[kYPlaneIndex] +
+ av_frame_->linesize[kYPlaneIndex] * av_frame_->height,
+ planar_yuv8_buffer->DataY() + planar_yuv8_buffer->StrideY() *
+ planar_yuv8_buffer->height());
+ RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], planar_yuv8_buffer->DataU());
RTC_DCHECK_LE(av_frame_->data[kUPlaneIndex] +
av_frame_->linesize[kUPlaneIndex] * av_frame_->height / 2,
- i420_buffer->DataU() +
- i420_buffer->StrideU() * i420_buffer->height() / 2);
- RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], i420_buffer->DataV());
+ planar_yuv8_buffer->DataU() + planar_yuv8_buffer->StrideU() *
+ planar_yuv8_buffer->height() /
+ 2);
+ RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], planar_yuv8_buffer->DataV());
RTC_DCHECK_LE(av_frame_->data[kVPlaneIndex] +
av_frame_->linesize[kVPlaneIndex] * av_frame_->height / 2,
- i420_buffer->DataV() +
- i420_buffer->StrideV() * i420_buffer->height() / 2);
+ planar_yuv8_buffer->DataV() + planar_yuv8_buffer->StrideV() *
+ planar_yuv8_buffer->height() /
+ 2);
- rtc::scoped_refptr<webrtc::VideoFrameBuffer> cropped_buffer = WrapI420Buffer(
- av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
- av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
- av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
- av_frame_->linesize[kVPlaneIndex],
- // To keep reference alive.
- [frame_buffer] {});
+ rtc::scoped_refptr<webrtc::VideoFrameBuffer> cropped_buffer;
+ if (video_frame_buffer_type == VideoFrameBuffer::Type::kI420) {
+ cropped_buffer = WrapI420Buffer(
+ av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
+ av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
+ av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
+ av_frame_->linesize[kVPlaneIndex],
+ // To keep reference alive.
+ [frame_buffer] {});
+ } else {
+ cropped_buffer = WrapI444Buffer(
+ av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
+ av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
+ av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
+ av_frame_->linesize[kVPlaneIndex],
+ // To keep reference alive.
+ [frame_buffer] {});
+ }
if (preferred_output_format_ == VideoFrameBuffer::Type::kNV12) {
- const I420BufferInterface* cropped_i420 = cropped_buffer->GetI420();
auto nv12_buffer = output_buffer_pool_.CreateNV12Buffer(
- cropped_i420->width(), cropped_i420->height());
- libyuv::I420ToNV12(cropped_i420->DataY(), cropped_i420->StrideY(),
- cropped_i420->DataU(), cropped_i420->StrideU(),
- cropped_i420->DataV(), cropped_i420->StrideV(),
- nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
- nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
- i420_buffer->width(), i420_buffer->height());
+ cropped_buffer->width(), cropped_buffer->height());
+
+ const PlanarYuv8Buffer* cropped_planar_yuv8_buffer = nullptr;
+ if (video_frame_buffer_type == VideoFrameBuffer::Type::kI420) {
+ cropped_planar_yuv8_buffer = cropped_buffer->GetI420();
+ libyuv::I420ToNV12(cropped_planar_yuv8_buffer->DataY(),
+ cropped_planar_yuv8_buffer->StrideY(),
+ cropped_planar_yuv8_buffer->DataU(),
+ cropped_planar_yuv8_buffer->StrideU(),
+ cropped_planar_yuv8_buffer->DataV(),
+ cropped_planar_yuv8_buffer->StrideV(),
+ nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
+ nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
+ planar_yuv8_buffer->width(),
+ planar_yuv8_buffer->height());
+ } else {
+ cropped_planar_yuv8_buffer = cropped_buffer->GetI444();
+ libyuv::I444ToNV12(cropped_planar_yuv8_buffer->DataY(),
+ cropped_planar_yuv8_buffer->StrideY(),
+ cropped_planar_yuv8_buffer->DataU(),
+ cropped_planar_yuv8_buffer->StrideU(),
+ cropped_planar_yuv8_buffer->DataV(),
+ cropped_planar_yuv8_buffer->StrideV(),
+ nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
+ nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
+ planar_yuv8_buffer->width(),
+ planar_yuv8_buffer->height());
+ }
+
cropped_buffer = nv12_buffer;
}
diff --git a/modules/video_coding/codecs/h264/include/h264.h b/modules/video_coding/codecs/h264/include/h264.h
index 8d1eebc..8c201d2 100644
--- a/modules/video_coding/codecs/h264/include/h264.h
+++ b/modules/video_coding/codecs/h264/include/h264.h
@@ -38,10 +38,15 @@
// and is not thread-safe.
RTC_EXPORT void DisableRtcUseH264();
-// Returns a vector with all supported internal H264 profiles that we can
+// Returns a vector with all supported internal H264 encode profiles that we can
// negotiate in SDP, in order of preference.
std::vector<SdpVideoFormat> SupportedH264Codecs();
+// Returns a vector with all supported internal H264 decode profiles that we can
+// negotiate in SDP, in order of preference. This will be available for receive
+// only connections.
+std::vector<SdpVideoFormat> SupportedH264DecoderCodecs();
+
class RTC_EXPORT H264Encoder : public VideoEncoder {
public:
static std::unique_ptr<H264Encoder> Create(const cricket::VideoCodec& codec);
diff --git a/pc/media_session.cc b/pc/media_session.cc
index 45cedfb..ed292cf 100644
--- a/pc/media_session.cc
+++ b/pc/media_session.cc
@@ -2902,8 +2902,12 @@
video_sendrecv_codecs_.clear();
all_video_codecs_.clear();
// Compute the video codecs union.
+ // Keep track of payload types to avoid collisions.
+ UsedPayloadTypes used_payload_types;
for (const VideoCodec& send : video_send_codecs_) {
- all_video_codecs_.push_back(send);
+ VideoCodec send_mutable = send;
+ used_payload_types.FindAndSetIdUsed(&send_mutable);
+ all_video_codecs_.push_back(send_mutable);
if (!FindMatchingCodec<VideoCodec>(video_send_codecs_, video_recv_codecs_,
send, nullptr)) {
// TODO(kron): This check is violated by the unit test:
@@ -2915,12 +2919,11 @@
// RTC_DCHECK(!IsRtxCodec(send));
}
}
- for (const VideoCodec& recv : video_recv_codecs_) {
- if (!FindMatchingCodec<VideoCodec>(video_recv_codecs_, video_send_codecs_,
- recv, nullptr)) {
- all_video_codecs_.push_back(recv);
- }
- }
+ // Use MergeCodecs to merge the second half of our list as it already checks
+ // and fixes problems with duplicate payload types.
+ MergeCodecs<VideoCodec>(video_recv_codecs_, &all_video_codecs_,
+ &used_payload_types);
+
// Use NegotiateCodecs to merge our codec lists, since the operation is
// essentially the same. Put send_codecs as the offered_codecs, which is the
// order we'd like to follow. The reasoning is that encoding is usually more
diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm
index 966eb3e..93ed03a 100644
--- a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm
+++ b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm
@@ -240,6 +240,7 @@
case webrtc::H264Profile::kProfileConstrainedHigh:
case webrtc::H264Profile::kProfileHigh:
+ case webrtc::H264Profile::kProfilePredictiveHigh444:
switch (profile_level_id.level) {
case webrtc::H264Level::kLevel3:
return kVTProfileLevel_H264_High_3_0;
