Add alpha channel to VideoFrameBuffer containers
- Add alpha accessors to PlanarYuvBuffer interface, null by defualt.
- Add WrapI420ABuffer() that creates a container which implements these
accessors.
- Show the use via StereoDecoderAdapter.
This CL is the step 2 for adding alpha channel support over the wire in webrtc.
See https://webrtc-review.googlesource.com/c/src/+/7800 for the experimental
CL that gives an idea about how it will come together.
Design Doc: https://goo.gl/sFeSUT
Bug: webrtc:7671
Change-Id: Id5691cde00088ec811b63d89080d33ad2d6e3939
Reviewed-on: https://webrtc-review.googlesource.com/21130
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Commit-Queue: Emircan Uysaler <emircan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20635}diff --git a/api/video/video_frame_buffer.cc b/api/video/video_frame_buffer.cc
index dd869f7..b231745 100644
--- a/api/video/video_frame_buffer.cc
+++ b/api/video/video_frame_buffer.cc
@@ -27,6 +27,16 @@
return static_cast<const I420BufferInterface*>(this);
}
+I420ABufferInterface* VideoFrameBuffer::GetI420A() {
+ RTC_CHECK(type() == Type::kI420A);
+ return static_cast<I420ABufferInterface*>(this);
+}
+
+const I420ABufferInterface* VideoFrameBuffer::GetI420A() const {
+ RTC_CHECK(type() == Type::kI420A);
+ return static_cast<const I420ABufferInterface*>(this);
+}
+
I444BufferInterface* VideoFrameBuffer::GetI444() {
RTC_CHECK(type() == Type::kI444);
return static_cast<I444BufferInterface*>(this);
@@ -53,6 +63,10 @@
return this;
}
+VideoFrameBuffer::Type I420ABufferInterface::type() const {
+ return Type::kI420A;
+}
+
VideoFrameBuffer::Type I444BufferInterface::type() const {
return Type::kI444;
}
diff --git a/api/video/video_frame_buffer.h b/api/video/video_frame_buffer.h
index c8d02a1..e656edd 100644
--- a/api/video/video_frame_buffer.h
+++ b/api/video/video_frame_buffer.h
@@ -19,6 +19,7 @@
namespace webrtc {
class I420BufferInterface;
+class I420ABufferInterface;
class I444BufferInterface;
// Base class for frame buffers of different types of pixel format and storage.
@@ -44,6 +45,7 @@
enum class Type {
kNative,
kI420,
+ kI420A,
kI444,
};
@@ -67,6 +69,8 @@
// removed.
rtc::scoped_refptr<I420BufferInterface> GetI420();
rtc::scoped_refptr<const I420BufferInterface> GetI420() const;
+ I420ABufferInterface* GetI420A();
+ const I420ABufferInterface* GetI420A() const;
I444BufferInterface* GetI444();
const I444BufferInterface* GetI444() const;
@@ -97,7 +101,7 @@
class I420BufferInterface : public PlanarYuvBuffer {
public:
- Type type() const final;
+ Type type() const override;
int ChromaWidth() const final;
int ChromaHeight() const final;
@@ -108,6 +112,16 @@
~I420BufferInterface() override {}
};
+class I420ABufferInterface : public I420BufferInterface {
+ public:
+ Type type() const final;
+ virtual const uint8_t* DataA() const = 0;
+ virtual int StrideA() const = 0;
+
+ protected:
+ ~I420ABufferInterface() override {}
+};
+
class I444BufferInterface : public PlanarYuvBuffer {
public:
Type type() const final;
diff --git a/common_video/include/video_frame_buffer.h b/common_video/include/video_frame_buffer.h
index bc9d4f9..312979b 100644
--- a/common_video/include/video_frame_buffer.h
+++ b/common_video/include/video_frame_buffer.h
@@ -78,6 +78,19 @@
int v_stride,
const rtc::Callback0<void>& no_longer_used);
+rtc::scoped_refptr<I420ABufferInterface> WrapI420ABuffer(
+ int width,
+ int height,
+ const uint8_t* y_plane,
+ int y_stride,
+ const uint8_t* u_plane,
+ int u_stride,
+ const uint8_t* v_plane,
+ int v_stride,
+ const uint8_t* a_plane,
+ int a_stride,
+ const rtc::Callback0<void>& no_longer_used);
+
rtc::scoped_refptr<PlanarYuvBuffer> WrapYuvBuffer(
VideoFrameBuffer::Type type,
int width,
diff --git a/common_video/video_frame_buffer.cc b/common_video/video_frame_buffer.cc
index 19649cf..9c8955c 100644
--- a/common_video/video_frame_buffer.cc
+++ b/common_video/video_frame_buffer.cc
@@ -96,6 +96,8 @@
v_stride_(v_stride),
no_longer_used_cb_(no_longer_used) {}
+ ~WrappedYuvBuffer() override { no_longer_used_cb_(); }
+
int width() const override { return width_; }
int height() const override { return height_; }
@@ -115,8 +117,6 @@
private:
friend class rtc::RefCountedObject<WrappedYuvBuffer>;
- ~WrappedYuvBuffer() override { no_longer_used_cb_(); }
-
const int width_;
const int height_;
const uint8_t* const y_plane_;
@@ -128,6 +128,41 @@
rtc::Callback0<void> no_longer_used_cb_;
};
+// Template to implement a wrapped buffer for a I4??BufferInterface.
+template <typename BaseWithA>
+class WrappedYuvaBuffer : public WrappedYuvBuffer<BaseWithA> {
+ public:
+ WrappedYuvaBuffer(int width,
+ int height,
+ const uint8_t* y_plane,
+ int y_stride,
+ const uint8_t* u_plane,
+ int u_stride,
+ const uint8_t* v_plane,
+ int v_stride,
+ const uint8_t* a_plane,
+ int a_stride,
+ const rtc::Callback0<void>& no_longer_used)
+ : WrappedYuvBuffer<BaseWithA>(width,
+ height,
+ y_plane,
+ y_stride,
+ u_plane,
+ u_stride,
+ v_plane,
+ v_stride,
+ no_longer_used),
+ a_plane_(a_plane),
+ a_stride_(a_stride) {}
+
+ const uint8_t* DataA() const override { return a_plane_; }
+ int StrideA() const override { return a_stride_; }
+
+ private:
+ const uint8_t* const a_plane_;
+ const int a_stride_;
+};
+
rtc::scoped_refptr<I420BufferInterface> WrapI420Buffer(
int width,
int height,
@@ -144,6 +179,24 @@
v_stride, no_longer_used));
}
+rtc::scoped_refptr<I420ABufferInterface> WrapI420ABuffer(
+ int width,
+ int height,
+ const uint8_t* y_plane,
+ int y_stride,
+ const uint8_t* u_plane,
+ int u_stride,
+ const uint8_t* v_plane,
+ int v_stride,
+ const uint8_t* a_plane,
+ int a_stride,
+ const rtc::Callback0<void>& no_longer_used) {
+ return rtc::scoped_refptr<I420ABufferInterface>(
+ new rtc::RefCountedObject<WrappedYuvaBuffer<I420ABufferInterface>>(
+ width, height, y_plane, y_stride, u_plane, u_stride, v_plane,
+ v_stride, a_plane, a_stride, no_longer_used));
+}
+
rtc::scoped_refptr<I444BufferInterface> WrapI444Buffer(
int width,
int height,
diff --git a/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc b/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc
index 5f8b4c59..82e87d4 100644
--- a/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc
+++ b/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc
@@ -11,6 +11,7 @@
#include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h"
#include "api/video/i420_buffer.h"
+#include "api/video/video_frame_buffer.h"
#include "api/video_codecs/sdp_video_format.h"
#include "common_video/include/video_frame.h"
#include "common_video/include/video_frame_buffer.h"
@@ -18,6 +19,11 @@
#include "rtc_base/keep_ref_until_done.h"
#include "rtc_base/logging.h"
+namespace {
+void KeepBufferRefs(rtc::scoped_refptr<webrtc::VideoFrameBuffer>,
+ rtc::scoped_refptr<webrtc::VideoFrameBuffer>) {}
+} // anonymous namespace
+
namespace webrtc {
class StereoDecoderAdapter::AdapterDecodedImageCallback
@@ -173,11 +179,22 @@
VideoFrame* alpha_decodedImage,
const rtc::Optional<int32_t>& alpha_decode_time_ms,
const rtc::Optional<uint8_t>& alpha_qp) {
- // TODO(emircan): Merge the output and put in a VideoFrame container that can
- // transport I420A.
- decoded_complete_callback_->Decoded(*decodedImage, decode_time_ms, qp);
- decoded_complete_callback_->Decoded(*alpha_decodedImage, alpha_decode_time_ms,
- alpha_qp);
+ rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
+ decodedImage->video_frame_buffer()->ToI420();
+ rtc::scoped_refptr<webrtc::I420BufferInterface> alpha_buffer =
+ alpha_decodedImage->video_frame_buffer()->ToI420();
+ RTC_DCHECK_EQ(yuv_buffer->width(), alpha_buffer->width());
+ RTC_DCHECK_EQ(yuv_buffer->height(), alpha_buffer->height());
+ rtc::scoped_refptr<I420ABufferInterface> merged_buffer = WrapI420ABuffer(
+ yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
+ yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
+ yuv_buffer->DataV(), yuv_buffer->StrideV(), alpha_buffer->DataY(),
+ alpha_buffer->StrideY(),
+ rtc::Bind(&KeepBufferRefs, yuv_buffer, alpha_buffer));
+
+ VideoFrame merged_image(merged_buffer, decodedImage->timestamp(),
+ 0 /* render_time_ms */, decodedImage->rotation());
+ decoded_complete_callback_->Decoded(merged_image, decode_time_ms, qp);
}
} // namespace webrtc
diff --git a/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc b/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc
index 283273d..b449c68 100644
--- a/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc
+++ b/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc
@@ -83,28 +83,30 @@
if (!encoded_complete_callback_) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
-
- // TODO(emircan): Extract alpha and create an alpha frame with dummy planes.
- // Since we don't have a way of transporting alpha yet, put a dummy output for
- // alpha consisting of YXX.
-
- // Encode AXX
- rtc::scoped_refptr<I420BufferInterface> yuva_buffer =
- input_image.video_frame_buffer()->ToI420();
- rtc::scoped_refptr<WrappedI420Buffer> alpha_buffer(
- new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
- input_image.width(), input_image.height(), yuva_buffer->DataY(),
- yuva_buffer->StrideY(), stereo_dummy_planes_.data(),
- yuva_buffer->StrideU(), stereo_dummy_planes_.data(),
- yuva_buffer->StrideV(),
- rtc::KeepRefUntilDone(input_image.video_frame_buffer())));
- VideoFrame alpha_image(alpha_buffer, input_image.timestamp(),
- input_image.render_time_ms(), input_image.rotation());
- encoders_[kAXXStream]->Encode(alpha_image, codec_specific_info, frame_types);
-
// Encode YUV
int rv = encoders_[kYUVStream]->Encode(input_image, codec_specific_info,
frame_types);
+ if (rv)
+ return rv;
+
+ const bool has_alpha = input_image.video_frame_buffer()->type() ==
+ VideoFrameBuffer::Type::kI420A;
+ if (!has_alpha)
+ return rv;
+
+ // Encode AXX
+ const I420ABufferInterface* yuva_buffer =
+ input_image.video_frame_buffer()->GetI420A();
+ rtc::scoped_refptr<I420BufferInterface> alpha_buffer =
+ WrapI420Buffer(input_image.width(), input_image.height(),
+ yuva_buffer->DataA(), yuva_buffer->StrideA(),
+ stereo_dummy_planes_.data(), yuva_buffer->StrideU(),
+ stereo_dummy_planes_.data(), yuva_buffer->StrideV(),
+ rtc::KeepRefUntilDone(input_image.video_frame_buffer()));
+ VideoFrame alpha_image(alpha_buffer, input_image.timestamp(),
+ input_image.render_time_ms(), input_image.rotation());
+ rv = encoders_[kAXXStream]->Encode(alpha_image, codec_specific_info,
+ frame_types);
return rv;
}
@@ -158,6 +160,9 @@
const EncodedImage& encodedImage,
const CodecSpecificInfo* codecSpecificInfo,
const RTPFragmentationHeader* fragmentation) {
+ if (stream_idx == kAXXStream)
+ return EncodedImageCallback::Result(EncodedImageCallback::Result::OK);
+
// TODO(emircan): Fill |codec_specific_info| with stereo parameters.
encoded_complete_callback_->OnEncodedImage(encodedImage, codecSpecificInfo,
fragmentation);
diff --git a/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc b/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc
index a8913e7..7c16620 100644
--- a/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc
+++ b/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc
@@ -10,11 +10,14 @@
#include "api/test/mock_video_decoder_factory.h"
#include "api/test/mock_video_encoder_factory.h"
+#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h"
#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h"
#include "modules/video_coding/codecs/test/video_codec_test.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
+#include "rtc_base/keep_ref_until_done.h"
+#include "rtc_base/ptr_util.h"
using testing::_;
using testing::Return;
@@ -44,6 +47,18 @@
return codec_settings;
}
+ std::unique_ptr<VideoFrame> CreateI420AInputFrame() {
+ rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
+ input_frame_->video_frame_buffer()->ToI420();
+ rtc::scoped_refptr<I420ABufferInterface> yuva_buffer = WrapI420ABuffer(
+ yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
+ yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
+ yuv_buffer->DataV(), yuv_buffer->StrideV(), yuv_buffer->DataY(),
+ yuv_buffer->StrideY(), rtc::KeepRefUntilDone(yuv_buffer));
+ return rtc::WrapUnique<VideoFrame>(
+ new VideoFrame(yuva_buffer, kVideoRotation_0, 0));
+ }
+
private:
void SetUp() override {
EXPECT_CALL(*decoder_factory_, Die());
@@ -93,4 +108,20 @@
EXPECT_GT(I420PSNR(input_frame_.get(), decoded_frame.get()), 36);
}
+TEST_F(TestStereoAdapter, EncodeDecodeI420AFrame) {
+ std::unique_ptr<VideoFrame> yuva_frame = CreateI420AInputFrame();
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(*yuva_frame, nullptr, nullptr));
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ decoder_->Decode(encoded_frame, false, nullptr));
+ std::unique_ptr<VideoFrame> decoded_frame;
+ rtc::Optional<uint8_t> decoded_qp;
+ ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
+ ASSERT_TRUE(decoded_frame);
+ EXPECT_GT(I420PSNR(yuva_frame.get(), decoded_frame.get()), 36);
+}
+
} // namespace webrtc
