Add AV1 profile-1 video decode support to WebRTC
The Chrome Remote Desktop team is looking to support AV1 profile-1
w/ I444 for screen sharing however only I420 is currently supported.
This CL adds I444 support for the Dav1dDecoder, which appears to be
the preferred decoder and adds profile-1 to the
InternalDecoderFactory when the Dav1dDecoder is being used.
I've tested this CL using a CRD host w/ I444 enabled and it seems to
work as expected, though I've only tested on a debug build so I plan
to do some perf testing once this is available in a release build.
Bug: chromium:1329660
Change-Id: I2b8b7b7fd530727456ac5c46e694e7dbad6deff2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/273986
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Joe Downing <joedow@google.com>
Cr-Commit-Position: refs/heads/main@{#38022}
diff --git a/media/engine/internal_decoder_factory.cc b/media/engine/internal_decoder_factory.cc
index fb8510b..4119d73 100644
--- a/media/engine/internal_decoder_factory.cc
+++ b/media/engine/internal_decoder_factory.cc
@@ -11,6 +11,7 @@
#include "media/engine/internal_decoder_factory.h"
#include "absl/strings/match.h"
+#include "api/video_codecs/av1_profile.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_codec.h"
#include "media/base/codec.h"
@@ -50,10 +51,16 @@
for (const SdpVideoFormat& h264_format : SupportedH264DecoderCodecs())
formats.push_back(h264_format);
- if (kIsLibaomAv1DecoderSupported ||
- (kDav1dIsIncluded && !field_trial::IsDisabled(kDav1dFieldTrial))) {
+ bool isDav1dEnabled =
+ kDav1dIsIncluded && !field_trial::IsDisabled(kDav1dFieldTrial);
+ if (kIsLibaomAv1DecoderSupported || isDav1dEnabled) {
formats.push_back(SdpVideoFormat(cricket::kAv1CodecName));
}
+ if (isDav1dEnabled) {
+ formats.push_back(SdpVideoFormat(
+ cricket::kAv1CodecName,
+ {{kAV1FmtpProfile, AV1ProfileToString(AV1Profile::kProfile1).data()}}));
+ }
return formats;
}
diff --git a/media/engine/internal_decoder_factory_unittest.cc b/media/engine/internal_decoder_factory_unittest.cc
index d70390f..e2789af 100644
--- a/media/engine/internal_decoder_factory_unittest.cc
+++ b/media/engine/internal_decoder_factory_unittest.cc
@@ -10,11 +10,13 @@
#include "media/engine/internal_decoder_factory.h"
+#include "api/video_codecs/av1_profile.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/vp9_profile.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
+#include "system_wrappers/include/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -24,6 +26,8 @@
using ::testing::Field;
using ::testing::Not;
+using ::webrtc::field_trial::InitFieldTrialsFromString;
+
#ifdef RTC_ENABLE_VP9
constexpr bool kVp9Enabled = true;
#else
@@ -34,10 +38,19 @@
#else
constexpr bool kH264Enabled = false;
#endif
+#ifdef RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY
+constexpr bool kDav1dIsIncluded = true;
+#else
+constexpr bool kDav1dIsIncluded = false;
+#endif
constexpr VideoDecoderFactory::CodecSupport kSupported = {
/*is_supported=*/true, /*is_power_efficient=*/false};
constexpr VideoDecoderFactory::CodecSupport kUnsupported = {
/*is_supported=*/false, /*is_power_efficient=*/false};
+constexpr char kDav1dDecoderFieldTrialEnabled[] =
+ "WebRTC-Dav1dDecoder/Enabled/";
+constexpr char kDav1dDecoderFieldTrialDisabled[] =
+ "WebRTC-Dav1dDecoder/Disabled/";
MATCHER_P(Support, expected, "") {
return arg.is_supported == expected.is_supported &&
@@ -76,9 +89,10 @@
EXPECT_EQ(static_cast<bool>(decoder), kH264Enabled);
}
-TEST(InternalDecoderFactoryTest, Av1) {
+TEST(InternalDecoderFactoryTest, Av1Profile0) {
InternalDecoderFactory factory;
- if (kIsLibaomAv1DecoderSupported) {
+ InitFieldTrialsFromString(kDav1dDecoderFieldTrialEnabled);
+ if (kIsLibaomAv1DecoderSupported || kDav1dIsIncluded) {
EXPECT_THAT(factory.GetSupportedFormats(),
Contains(Field(&SdpVideoFormat::name, cricket::kAv1CodecName)));
EXPECT_TRUE(
@@ -90,6 +104,26 @@
}
}
+TEST(InternalDecoderFactoryTest, Av1Profile1_Dav1dDecoderTrialEnabled) {
+ InitFieldTrialsFromString(kDav1dDecoderFieldTrialEnabled);
+ InternalDecoderFactory factory;
+ std::unique_ptr<VideoDecoder> decoder = factory.CreateVideoDecoder(
+ SdpVideoFormat(cricket::kAv1CodecName,
+ {{kAV1FmtpProfile,
+ AV1ProfileToString(AV1Profile::kProfile1).data()}}));
+ EXPECT_EQ(static_cast<bool>(decoder), kDav1dIsIncluded);
+}
+
+TEST(InternalDecoderFactoryTest, Av1Profile1_Dav1dDecoderTrialDisabled) {
+ InitFieldTrialsFromString(kDav1dDecoderFieldTrialDisabled);
+ InternalDecoderFactory factory;
+ std::unique_ptr<VideoDecoder> decoder = factory.CreateVideoDecoder(
+ SdpVideoFormat(cricket::kAv1CodecName,
+ {{kAV1FmtpProfile,
+ AV1ProfileToString(AV1Profile::kProfile1).data()}}));
+ EXPECT_FALSE(static_cast<bool>(decoder));
+}
+
TEST(InternalDecoderFactoryTest, QueryCodecSupportNoReferenceScaling) {
InternalDecoderFactory factory;
EXPECT_THAT(factory.QueryCodecSupport(SdpVideoFormat(cricket::kVp8CodecName),
diff --git a/modules/video_coding/codecs/av1/dav1d_decoder.cc b/modules/video_coding/codecs/av1/dav1d_decoder.cc
index a5e4784..4b62216 100644
--- a/modules/video_coding/codecs/av1/dav1d_decoder.cc
+++ b/modules/video_coding/codecs/av1/dav1d_decoder.cc
@@ -14,12 +14,13 @@
#include "api/scoped_refptr.h"
#include "api/video/encoded_image.h"
-#include "api/video/i420_buffer.h"
+#include "api/video/video_frame_buffer.h"
#include "common_video/include/video_frame_buffer_pool.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/logging.h"
#include "third_party/dav1d/libdav1d/include/dav1d/dav1d.h"
#include "third_party/libyuv/include/libyuv/convert.h"
+#include "third_party/libyuv/include/libyuv/planar_functions.h"
namespace webrtc {
namespace {
@@ -70,6 +71,56 @@
constexpr char kDav1dName[] = "dav1d";
+rtc::scoped_refptr<VideoFrameBuffer> CopyFrameData(
+ VideoFrameBufferPool& buffer_pool,
+ const Dav1dPicture& dav1d_picture) {
+ // PlanarYuv8Buffer can represent both I420 and I444 frames.
+ rtc::scoped_refptr<PlanarYuv8Buffer> buffer;
+ if (dav1d_picture.p.layout == DAV1D_PIXEL_LAYOUT_I420) {
+ buffer = buffer_pool.CreateI420Buffer(dav1d_picture.p.w, dav1d_picture.p.h);
+ } else if (dav1d_picture.p.layout == DAV1D_PIXEL_LAYOUT_I444) {
+ buffer = buffer_pool.CreateI444Buffer(dav1d_picture.p.w, dav1d_picture.p.h);
+ } else {
+ RTC_DCHECK_NOTREACHED() << "Unsupported layout: " << dav1d_picture.p.layout;
+ }
+ if (!buffer.get()) {
+ RTC_LOG(LS_ERROR) << "Dav1dDecoder: failed to allocate a video frame.";
+ return nullptr;
+ }
+
+ uint8_t* src_y_data = static_cast<uint8_t*>(dav1d_picture.data[0]);
+ uint8_t* src_u_data = static_cast<uint8_t*>(dav1d_picture.data[1]);
+ uint8_t* src_v_data = static_cast<uint8_t*>(dav1d_picture.data[2]);
+ int src_y_stride = dav1d_picture.stride[0];
+ int src_uv_stride = dav1d_picture.stride[1];
+ uint8_t* dst_y_data = const_cast<uint8_t*>(buffer->DataY());
+ uint8_t* dst_u_data = const_cast<uint8_t*>(buffer->DataU());
+ uint8_t* dst_v_data = const_cast<uint8_t*>(buffer->DataV());
+
+ if (dav1d_picture.p.layout == DAV1D_PIXEL_LAYOUT_I420) {
+ libyuv::I420Copy(src_y_data, src_y_stride, //
+ src_u_data, src_uv_stride, //
+ src_v_data, src_uv_stride, //
+ dst_y_data, buffer->StrideY(), //
+ dst_u_data, buffer->StrideU(), //
+ dst_v_data, buffer->StrideV(), //
+ dav1d_picture.p.w, //
+ dav1d_picture.p.h); //
+ } else {
+ RTC_DCHECK_EQ(dav1d_picture.p.layout, DAV1D_PIXEL_LAYOUT_I444);
+ libyuv::I444Copy(src_y_data, src_y_stride, //
+ src_u_data, src_uv_stride, //
+ src_v_data, src_uv_stride, //
+ dst_y_data, buffer->StrideY(), //
+ dst_u_data, buffer->StrideU(), //
+ dst_v_data, buffer->StrideV(), //
+ dav1d_picture.p.w, //
+ dav1d_picture.p.h); //
+ }
+
+ return buffer;
+}
+
// Calling `dav1d_data_wrap` requires a `free_callback` to be registered.
void NullFreeCallback(const uint8_t* buffer, void* opaque) {}
@@ -147,34 +198,27 @@
return WEBRTC_VIDEO_CODEC_ERROR;
}
- // Only accept I420 pixel format and 8 bit depth.
- if (dav1d_picture.p.layout != DAV1D_PIXEL_LAYOUT_I420 ||
- dav1d_picture.p.bpc != 8) {
+ if (dav1d_picture.p.bpc != 8) {
+ // Only accept 8 bit depth.
+ RTC_LOG(LS_ERROR) << "Dav1dDecoder::Decode unhandled bit depth: "
+ << dav1d_picture.p.bpc;
return WEBRTC_VIDEO_CODEC_ERROR;
}
- rtc::scoped_refptr<I420Buffer> buffer =
- buffer_pool_.CreateI420Buffer(dav1d_picture.p.w, dav1d_picture.p.h);
- if (!buffer.get()) {
- RTC_LOG(LS_WARNING)
- << "Dav1dDecoder::Decode failed to get frame from the buffer pool.";
+ rtc::scoped_refptr<VideoFrameBuffer> buffer;
+ if (dav1d_picture.p.layout == DAV1D_PIXEL_LAYOUT_I420 ||
+ dav1d_picture.p.layout == DAV1D_PIXEL_LAYOUT_I444) {
+ buffer = CopyFrameData(buffer_pool_, dav1d_picture);
+ if (!buffer.get()) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ } else {
+ // Only accept I420 or I444 pixel format.
+ RTC_LOG(LS_ERROR) << "Dav1dDecoder::Decode unhandled pixel layout: "
+ << dav1d_picture.p.layout;
return WEBRTC_VIDEO_CODEC_ERROR;
}
- uint8_t* y_data = static_cast<uint8_t*>(dav1d_picture.data[0]);
- uint8_t* u_data = static_cast<uint8_t*>(dav1d_picture.data[1]);
- uint8_t* v_data = static_cast<uint8_t*>(dav1d_picture.data[2]);
- int y_stride = dav1d_picture.stride[0];
- int uv_stride = dav1d_picture.stride[1];
- libyuv::I420Copy(y_data, y_stride, //
- u_data, uv_stride, //
- v_data, uv_stride, //
- buffer->MutableDataY(), buffer->StrideY(), //
- buffer->MutableDataU(), buffer->StrideU(), //
- buffer->MutableDataV(), buffer->StrideV(), //
- dav1d_picture.p.w, //
- dav1d_picture.p.h); //
-
VideoFrame decoded_frame = VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp(encoded_image.Timestamp())
